diff --git a/contracts/cultmesh/content-acceptance-vectors.json b/contracts/cultmesh/content-acceptance-vectors.json index a938f430..d9868a63 100644 --- a/contracts/cultmesh/content-acceptance-vectors.json +++ b/contracts/cultmesh/content-acceptance-vectors.json @@ -4,7 +4,7 @@ "label": "base", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { @@ -67,112 +67,112 @@ "label": "short_8", "inputBase64": "mKNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWs=", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuQgA==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkIA=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "short_9", "inputBase64": "maNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXg=", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiA", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gA==", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "array16_header", "inputBase64": "3AAKo2EvYqVhc3NldKEx2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AK9hcHBsaWNhdGlvbi94LXShdJGU2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AACha5GheIGha6F2", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "array32_header", "inputBase64": "3QAAAAqjYS9ipWFzc2V0oTHZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAr2FwcGxpY2F0aW9uL3gtdKF0kZTZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAAKFrkaF4gaFroXY=", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "extra_11_nil", "inputBase64": "m6NhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdsA=", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "extra_11_str", "inputBase64": "m6NhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdqF4", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "extra_11_nested", "inputBase64": "m6NhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdpKRoXiBoWHLAAAAAAAAAAA=", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "extra_11_ext", "inputBase64": "m6NhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdtQBAg==", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "extra_11_ext8", "inputBase64": "m6NhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdscCAQcH", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "extra_11_bin", "inputBase64": "m6NhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdsQCAQI=", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "extra_11_float", "inputBase64": "m6NhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdsoAAAAA", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "extra_11_uint64", "inputBase64": "m6NhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhds8AAAAAAAAABw==", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "extra_11_array16", "inputBase64": "m6NhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdtwAAgEC", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "extra_11_map16", "inputBase64": "m6NhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdt4AAQEC", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "extra_11_str8", "inputBase64": "m6NhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdtkBQQ==", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "extra_12", "inputBase64": "nKNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdsCheQ==", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { @@ -193,40 +193,40 @@ "label": "deep_extra_250", "inputBase64": "m6NhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdpGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZHA", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "deep_chunkextra_250", - "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRwJGheIGha6F2", + "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRltlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMJGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZHAkaF4gaFroXY=", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "deep_extra_253", "inputBase64": "m6NhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdpGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZHA", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "deep_chunkextra_253", - "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRwJGheIGha6F2", + "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRltlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMJGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZHAkaF4gaFroXY=", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "deep_extra_254", "inputBase64": "m6NhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdpGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRwA==", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "deep_chunkextra_254", - "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkcCRoXiBoWuhdg==", + "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRltlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMJGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRwJGheIGha6F2", "accepted": false, "reencodedBase64": null, "recordKey": null @@ -235,12 +235,12 @@ "label": "deep_extra_255", "inputBase64": "m6NhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdpGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkcA=", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "deep_chunkextra_255", - "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZHAkaF4gaFroXY=", + "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRltlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMJGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkcCRoXiBoWuhdg==", "accepted": false, "reencodedBase64": null, "recordKey": null @@ -254,7 +254,7 @@ }, { "label": "deep_chunkextra_256", - "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRwJGheIGha6F2", + "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRltlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMJGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZHAkaF4gaFroXY=", "accepted": false, "reencodedBase64": null, "recordKey": null @@ -268,7 +268,7 @@ }, { "label": "deep_chunkextra_257", - "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkcCRoXiBoWuhdg==", + "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRltlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMJGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRwJGheIGha6F2", "accepted": false, "reencodedBase64": null, "recordKey": null @@ -282,7 +282,7 @@ }, { "label": "deep_chunkextra_300", - "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZHAkaF4gaFroXY=", + "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRltlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMJGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkcCRoXiBoWuhdg==", "accepted": false, "reencodedBase64": null, "recordKey": null @@ -291,329 +291,357 @@ "label": "chunk_3", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRk9lAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAkaF4gaFroXY=", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoJGheIGha6F2", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoKCRoXiBoWuhdg==", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "chunk_5", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWvAkaF4gaFroXY=", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWvAkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "chunk_5_nested", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuSAQKRoXiBoWuhdg==", + "accepted": false, + "reencodedBase64": null, + "recordKey": null + }, + { + "label": "chunk_id_str", + "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMJGheIGha6F2", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMJGheIGha6F2", + "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "label": "chunk_id_int", + "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWsAkaF4gaFroXY=", + "accepted": false, + "reencodedBase64": null, + "recordKey": null + }, + { + "label": "chunk_id_str8", + "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWvZATCRoXiBoWuhdg==", + "accepted": true, + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMJGheIGha6F2", + "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "label": "chunk_6", + "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRltlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMMCRoXiBoWuhdg==", + "accepted": true, + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "chunk_extra_map", - "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuCoWGhYqFjoWSRoXiBoWuhdg==", + "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRltlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMIKhYaFioWOhZJGheIGha6F2", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "chunk_extra_map16", - "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWveAAIBAgMEkaF4gaFroXY=", + "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRltlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMN4AAgECAwSRoXiBoWuhdg==", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "chunk_extra_map32", - "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWvfAAAAAQECkaF4gaFroXY=", + "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRltlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMN8AAAABAQKRoXiBoWuhdg==", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "chunk_extra_array", - "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuTAQIDkaF4gaFroXY=", + "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRltlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMJMBAgORoXiBoWuhdg==", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "chunk_extra_array16", - "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWvcAAIBApGheIGha6F2", + "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRltlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMNwAAgECkaF4gaFroXY=", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "chunk_extra_array32", - "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWvdAAAAAQGRoXiBoWuhdg==", + "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRltlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMN0AAAABAZGheIGha6F2", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "chunk_extra_fixext1", - "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWvUAQKRoXiBoWuhdg==", + "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRltlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMNQBApGheIGha6F2", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "chunk_extra_fixext2", - "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWvVAQIDkaF4gaFroXY=", + "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRltlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMNUBAgORoXiBoWuhdg==", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "chunk_extra_fixext4", - "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWvWAQIDBAWRoXiBoWuhdg==", + "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRltlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMNYBAgMEBZGheIGha6F2", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "chunk_extra_fixext8", - "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWvXAQIDBAUGBwgJkaF4gaFroXY=", + "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRltlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMNcBAgMEBQYHCAmRoXiBoWuhdg==", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "chunk_extra_fixext16", - "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWvYAQAAAAAAAAAAAAAAAAAAAACRoXiBoWuhdg==", + "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRltlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMNgBAAAAAAAAAAAAAAAAAAAAAJGheIGha6F2", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "chunk_extra_ext8", - "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWvHAgEHB5GheIGha6F2", + "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRltlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMMcCAQcHkaF4gaFroXY=", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "chunk_extra_ext16", - "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWvIAAIBBweRoXiBoWuhdg==", + "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRltlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMMgAAgEHB5GheIGha6F2", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "chunk_extra_ext32", - "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWvJAAAAAgEHB5GheIGha6F2", + "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRltlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMMkAAAACAQcHkaF4gaFroXY=", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "chunk_extra_bin8", - "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWvEAgECkaF4gaFroXY=", + "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRltlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMMQCAQKRoXiBoWuhdg==", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "chunk_extra_bin16", - "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWvFAAIBApGheIGha6F2", + "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRltlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMMUAAgECkaF4gaFroXY=", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "chunk_extra_bin32", - "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWvGAAAAAgECkaF4gaFroXY=", + "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRltlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMMYAAAACAQKRoXiBoWuhdg==", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "chunk_extra_str8", - "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWvZAkFCkaF4gaFroXY=", + "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRltlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMNkCQUKRoXiBoWuhdg==", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "chunk_extra_str16", - "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWvaAAJBQpGheIGha6F2", + "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRltlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMNoAAkFCkaF4gaFroXY=", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "chunk_extra_str32", - "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWvbAAAAAkFCkaF4gaFroXY=", + "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRltlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMNsAAAACQUKRoXiBoWuhdg==", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "chunk_extra_fixstr", - "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuiYWKRoXiBoWuhdg==", + "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRltlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMKJhYpGheIGha6F2", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "chunk_extra_f32", - "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWvKAAAAAZGheIGha6F2", + "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRltlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMMoAAAABkaF4gaFroXY=", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "chunk_extra_f64", - "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWvLAAAAAAAAAAGRoXiBoWuhdg==", + "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRltlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMMsAAAAAAAAAAZGheIGha6F2", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "chunk_extra_u8", - "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWvMAZGheIGha6F2", + "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRltlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMMwBkaF4gaFroXY=", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "chunk_extra_u16", - "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWvNAAGRoXiBoWuhdg==", + "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRltlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMM0AAZGheIGha6F2", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "chunk_extra_u32", - "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWvOAAAAAZGheIGha6F2", + "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRltlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMM4AAAABkaF4gaFroXY=", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "chunk_extra_u64", - "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWvPAAAAAAAAAAGRoXiBoWuhdg==", + "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRltlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMM8AAAAAAAAAAZGheIGha6F2", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "chunk_extra_i8", - "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWvQAZGheIGha6F2", + "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRltlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMNABkaF4gaFroXY=", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "chunk_extra_i16", - "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWvRAAGRoXiBoWuhdg==", + "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRltlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMNEAAZGheIGha6F2", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "chunk_extra_i32", - "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWvSAAAAAZGheIGha6F2", + "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRltlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMNIAAAABkaF4gaFroXY=", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "chunk_extra_i64", - "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWvTAAAAAAAAAAGRoXiBoWuhdg==", + "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRltlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMNMAAAAAAAAAAZGheIGha6F2", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "chunk_extra_true", - "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWvDkaF4gaFroXY=", + "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRltlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMMORoXiBoWuhdg==", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "chunk_extra_false", - "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWvCkaF4gaFroXY=", + "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRltlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMMKRoXiBoWuhdg==", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "chunk_extra_nil", - "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWvAkaF4gaFroXY=", + "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRltlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMMCRoXiBoWuhdg==", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "chunk_extra_negfixint", - "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWv/kaF4gaFroXY=", + "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRltlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMP\u002BRoXiBoWuhdg==", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "chunk_extra_reserved", - "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWvBkaF4gaFroXY=", + "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRltlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMMGRoXiBoWuhdg==", "accepted": false, "reencodedBase64": null, "recordKey": null }, { "label": "chunk_extra_fixint_max", - "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWt/kaF4gaFroXY=", + "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRltlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMH\u002BRoXiBoWuhdg==", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "chunk_extra_negfixint_min", - "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWvgkaF4gaFroXY=", + "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRltlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMOCRoXiBoWuhdg==", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "chunk_extra_fixstr_31", - "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWu/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAJGheIGha6F2", + "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRltlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhML8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAkaF4gaFroXY=", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "chunk_extra_fixmap_15", - "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuPAQIBAgECAQIBAgECAQIBAgECAQIBAgECAQIBAgECkaF4gaFroXY=", + "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRltlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMI8BAgECAQIBAgECAQIBAgECAQIBAgECAQIBAgECAQKRoXiBoWuhdg==", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "chunk_extra_fixarray_15", - "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWufAAAAAAAAAAAAAAAAAAAAkaF4gaFroXY=", + "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRltlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMJ8AAAAAAAAAAAAAAAAAAACRoXiBoWuhdg==", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "chunk_extra_nested", - "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuSgaFhkgECkJGheIGha6F2", + "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRltlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMJKBoWGSAQKQkaF4gaFroXY=", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuhMJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "chunk_1", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRkdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NZGheIGha6F2", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoJGheIGha6F2", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoKCRoXiBoWuhdg==", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "chunk_0", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRkJGheIGha6F2", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlKAAAKCRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlaAAAKCgkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { @@ -669,42 +697,42 @@ "label": "nil_artifactId", "inputBase64": "msClYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", "accepted": true, - "reencodedBase64": "msClYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "msClYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=", "recordKey": null }, { "label": "nil_kind", "inputBase64": "mqNhL2LAoTHZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAr2FwcGxpY2F0aW9uL3gtdKF0kZTZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAAKFrkaF4gaFroXY=", "accepted": true, - "reencodedBase64": "mqNhL2LAoTHZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAr2FwcGxpY2F0aW9uL3gtdKF0kZTZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAAKFrkaF4gaFroXY=", + "reencodedBase64": "mqNhL2LAoTHZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAr2FwcGxpY2F0aW9uL3gtdKF0kZXZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAAKFroJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "nil_version", "inputBase64": "mqNhL2KlYXNzZXTA2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AK9hcHBsaWNhdGlvbi94LXShdJGU2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AACha5GheIGha6F2", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXTA2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AK9hcHBsaWNhdGlvbi94LXShdJGU2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AACha5GheIGha6F2", + "reencodedBase64": "mqNhL2KlYXNzZXTA2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AK9hcHBsaWNhdGlvbi94LXShdJGV2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AACha6CRoXiBoWuhdg==", "recordKey": "mesh:cdn:artifact:a-b:unversioned:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "nil_contentHash", "inputBase64": "mqNhL2KlYXNzZXShMcAAr2FwcGxpY2F0aW9uL3gtdKF0kZTZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAAKFrkaF4gaFroXY=", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMcAAr2FwcGxpY2F0aW9uL3gtdKF0kZTZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAAKFrkaF4gaFroXY=", + "reencodedBase64": "mqNhL2KlYXNzZXShMcAAr2FwcGxpY2F0aW9uL3gtdKF0kZXZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAAKFroJGheIGha6F2", "recordKey": null }, { "label": "nil_mimeType", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQDAoXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQDAoXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQDAoXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "nil_createdAtUtc", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10wJGU2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AACha5GheIGha6F2", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10wJGU2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AACha5GheIGha6F2", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10wJGV2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AACha6CRoXiBoWuhdg==", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { @@ -725,42 +753,42 @@ "label": "nil_chunk_hash", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlMAAAKFrkaF4gaFroXY=", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlMAAAKFrkaF4gaFroXY=", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlcAAAKFroJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "nil_chunk_recordKey", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAwJGheIGha6F2", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAwJGheIGha6F2", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAwKCRoXiBoWuhdg==", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "nil_tags", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWvAgaFroXY=", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWvAgaFroXY=", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugwIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "nil_tag_element", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuSwKF4gaFroXY=", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuSwKF4gaFroXY=", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugksCheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "nil_metadata", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXjA", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXjA", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4wA==", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "nil_metadata_value", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWvA", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWvA", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFrwA==", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { @@ -788,7 +816,7 @@ "label": "metadata_integer_like_keys", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiDojEwoWGlYWxwaGGhYqEyoWM=", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiDojEwoWGlYWxwaGGhYqEyoWM=", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4g6IxMKFhpWFscGhhoWKhMqFj", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { @@ -809,28 +837,28 @@ "label": "metadata_empty_key", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoKF2", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoKF2", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaChdg==", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "metadata_proto_key", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBqV9fcHJvdG9fX6F2", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBqV9fcHJvdG9fX6F2", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4galfX3Byb3RvX1\u002Bhdg==", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "metadata_map16", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXjeAAGha6F2", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "metadata_map32", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXjfAAAAAaFroXY=", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { @@ -865,28 +893,28 @@ "label": "tags_array16", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWvcAAGheIGha6F2", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "trailing_nil", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdsA=", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "trailing_reserved_code", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdsE=", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "trailing_truncated_array", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdpI=", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { @@ -935,168 +963,168 @@ "label": "size_u8", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NcwAr2FwcGxpY2F0aW9uL3gtdKF0kZTZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAAKFrkaF4gaFroXY=", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "offset_u8", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NcwAAKFrkaF4gaFroXY=", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "chunksize_u8", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQDMAKFrkaF4gaFroXY=", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "size_u16", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc0AAK9hcHBsaWNhdGlvbi94LXShdJGU2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AACha5GheIGha6F2", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "offset_u16", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc0AAACha5GheIGha6F2", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "chunksize_u16", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQDNAACha5GheIGha6F2", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "size_u32", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc4AAAAAr2FwcGxpY2F0aW9uL3gtdKF0kZTZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAAKFrkaF4gaFroXY=", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "offset_u32", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc4AAAAAAKFrkaF4gaFroXY=", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "chunksize_u32", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQDOAAAAAKFrkaF4gaFroXY=", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "size_u64", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc8AAAAAAAAAAK9hcHBsaWNhdGlvbi94LXShdJGU2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AACha5GheIGha6F2", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "offset_u64", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc8AAAAAAAAAAACha5GheIGha6F2", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "chunksize_u64", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQDPAAAAAAAAAACha5GheIGha6F2", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "size_i8", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NdAAr2FwcGxpY2F0aW9uL3gtdKF0kZTZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAAKFrkaF4gaFroXY=", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "offset_i8", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NdAAAKFrkaF4gaFroXY=", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "chunksize_i8", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQDQAKFrkaF4gaFroXY=", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "size_i16", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NdEAAK9hcHBsaWNhdGlvbi94LXShdJGU2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AACha5GheIGha6F2", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "offset_i16", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NdEAAACha5GheIGha6F2", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "chunksize_i16", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQDRAACha5GheIGha6F2", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "size_i32", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NdIAAAAAr2FwcGxpY2F0aW9uL3gtdKF0kZTZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAAKFrkaF4gaFroXY=", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "offset_i32", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NdIAAAAAAKFrkaF4gaFroXY=", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "chunksize_i32", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQDSAAAAAKFrkaF4gaFroXY=", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "size_i64", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NdMAAAAAAAAAAK9hcHBsaWNhdGlvbi94LXShdJGU2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AACha5GheIGha6F2", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "offset_i64", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NdMAAAAAAAAAAACha5GheIGha6F2", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "chunksize_i64", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQDTAAAAAAAAAACha5GheIGha6F2", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { @@ -1425,7 +1453,7 @@ "label": "chunksize_int_max", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc5/////r2FwcGxpY2F0aW9uL3gtdKF0kZTZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAzn////\u002Bha5GheIGha6F2", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc5/////r2FwcGxpY2F0aW9uL3gtdKF0kZTZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAzn////\u002Bha5GheIGha6F2", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc5/////r2FwcGxpY2F0aW9uL3gtdKF0kZXZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAzn////\u002Bha6CRoXiBoWuhdg==", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { @@ -1453,14 +1481,14 @@ "label": "offset_beyond_int32", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc8AAAABf////a9hcHBsaWNhdGlvbi94LXShdJOU2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AM5/////oWuU2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1zn/////Of////6FrlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc7////\u002Bzn////\u002Bha5GheIGha6F2", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc8AAAABf////a9hcHBsaWNhdGlvbi94LXShdJOU2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AM5/////oWuU2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1zn/////Of////6FrlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc7////\u002Bzn////\u002Bha5GheIGha6F2", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc8AAAABf////a9hcHBsaWNhdGlvbi94LXShdJOV2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AM5/////oWugldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc5/////zn////\u002Bha6CV2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1zv////7Of////6FroJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "chunks_out_of_order", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQKvYXBwbGljYXRpb24veC10oXSSlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQEBoWuU2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AAGha5GheIGha6F2", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQKvYXBwbGljYXRpb24veC10oXSSlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQEBoWuU2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AAGha5GheIGha6F2", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQKvYXBwbGljYXRpb24veC10oXSSldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQEBoWugldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQABoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { @@ -1502,105 +1530,105 @@ "label": "str_str16", "inputBase64": "mqNhL2LaAAFBoTHZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAr2FwcGxpY2F0aW9uL3gtdKF0kZTZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAAKFrkaF4gaFroXY=", "accepted": true, - "reencodedBase64": "mqNhL2KhQaEx2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AK9hcHBsaWNhdGlvbi94LXShdJGU2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AACha5GheIGha6F2", + "reencodedBase64": "mqNhL2KhQaEx2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AK9hcHBsaWNhdGlvbi94LXShdJGV2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AACha6CRoXiBoWuhdg==", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "str_str32", "inputBase64": "mqNhL2LbAAAAAUGhMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", "accepted": true, - "reencodedBase64": "mqNhL2KhQaEx2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AK9hcHBsaWNhdGlvbi94LXShdJGU2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AACha5GheIGha6F2", + "reencodedBase64": "mqNhL2KhQaEx2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AK9hcHBsaWNhdGlvbi94LXShdJGV2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AACha6CRoXiBoWuhdg==", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "str_empty", "inputBase64": "mqNhL2KgoTHZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAr2FwcGxpY2F0aW9uL3gtdKF0kZTZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAAKFrkaF4gaFroXY=", "accepted": true, - "reencodedBase64": "mqNhL2KgoTHZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAr2FwcGxpY2F0aW9uL3gtdKF0kZTZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAAKFrkaF4gaFroXY=", + "reencodedBase64": "mqNhL2KgoTHZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAr2FwcGxpY2F0aW9uL3gtdKF0kZXZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAAKFroJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "str_invalid_utf8", "inputBase64": "mqNhL2Ki//6hMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", "accepted": true, - "reencodedBase64": "mqNhL2Km77\u002B977\u002B9oTHZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAr2FwcGxpY2F0aW9uL3gtdKF0kZTZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAAKFrkaF4gaFroXY=", + "reencodedBase64": "mqNhL2Km77\u002B977\u002B9oTHZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAr2FwcGxpY2F0aW9uL3gtdKF0kZXZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAAKFroJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "str_lone_surrogate_cesu", "inputBase64": "mqNhL2Kj7aCAoTHZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAr2FwcGxpY2F0aW9uL3gtdKF0kZTZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAAKFrkaF4gaFroXY=", "accepted": true, - "reencodedBase64": "mqNhL2Kp77\u002B977\u002B977\u002B9oTHZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAr2FwcGxpY2F0aW9uL3gtdKF0kZTZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAAKFrkaF4gaFroXY=", + "reencodedBase64": "mqNhL2Kp77\u002B977\u002B977\u002B9oTHZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAr2FwcGxpY2F0aW9uL3gtdKF0kZXZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAAKFroJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "str_overlong", "inputBase64": "mqNhL2KiwK\u002BhMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", "accepted": true, - "reencodedBase64": "mqNhL2Km77\u002B977\u002B9oTHZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAr2FwcGxpY2F0aW9uL3gtdKF0kZTZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAAKFrkaF4gaFroXY=", + "reencodedBase64": "mqNhL2Km77\u002B977\u002B9oTHZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAr2FwcGxpY2F0aW9uL3gtdKF0kZXZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAAKFroJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "str_truncated_sequence", "inputBase64": "mqNhL2Ki4oKhMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", "accepted": true, - "reencodedBase64": "mqNhL2Kj77\u002B9oTHZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAr2FwcGxpY2F0aW9uL3gtdKF0kZTZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAAKFrkaF4gaFroXY=", + "reencodedBase64": "mqNhL2Kj77\u002B9oTHZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAr2FwcGxpY2F0aW9uL3gtdKF0kZXZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAAKFroJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "str_bom", "inputBase64": "mqbvu79ib22lYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", "accepted": true, - "reencodedBase64": "mqbvu79ib22lYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqbvu79ib22lYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:bom:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "str_nul", "inputBase64": "mqNhAGKlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", "accepted": true, - "reencodedBase64": "mqNhAGKlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhAGKlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "id_invalid_utf8", "inputBase64": "mqH/pWFzc2V0oTHZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAr2FwcGxpY2F0aW9uL3gtdKF0kZTZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAAKFrkaF4gaFroXY=", "accepted": true, - "reencodedBase64": "mqPvv72lYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqPvv72lYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:83d544ccc223c057d2bf80d3f2a32982c32c3c0db8e2674820da5064783fb097:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "hash_invalid_utf8", "inputBase64": "mqNhL2KlYXNzZXShMaH/AK9hcHBsaWNhdGlvbi94LXShdJGU2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AACha5GheIGha6F2", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMaPvv70Ar2FwcGxpY2F0aW9uL3gtdKF0kZTZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAAKFrkaF4gaFroXY=", + "reencodedBase64": "mqNhL2KlYXNzZXShMaPvv70Ar2FwcGxpY2F0aW9uL3gtdKF0kZXZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAAKFroJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:\uFFFD" }, { "label": "version_invalid_utf8", "inputBase64": "mqNhL2KlYXNzZXSi//7ZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAr2FwcGxpY2F0aW9uL3gtdKF0kZTZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAAKFrkaF4gaFroXY=", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXSm77\u002B977\u002B92UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AK9hcHBsaWNhdGlvbi94LXShdJGU2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AACha5GheIGha6F2", + "reencodedBase64": "mqNhL2KlYXNzZXSm77\u002B977\u002B92UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AK9hcHBsaWNhdGlvbi94LXShdJGV2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AACha6CRoXiBoWuhdg==", "recordKey": "mesh:cdn:artifact:a-b:52793f8dc1d85e409f8c88be99d8b31d58f676246340150f406289e04a11151e:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "id_blank", "inputBase64": "mqIgIKVhc3NldKEx2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AK9hcHBsaWNhdGlvbi94LXShdJGU2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AACha5GheIGha6F2", "accepted": true, - "reencodedBase64": "mqIgIKVhc3NldKEx2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AK9hcHBsaWNhdGlvbi94LXShdJGU2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AACha5GheIGha6F2", + "reencodedBase64": "mqIgIKVhc3NldKEx2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AK9hcHBsaWNhdGlvbi94LXShdJGV2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AACha6CRoXiBoWuhdg==", "recordKey": null }, { "label": "hash_blank", "inputBase64": "mqNhL2KlYXNzZXShMaIgIACvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMaIgIACvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMaIgIACvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=", "recordKey": null }, { "label": "hash_prefix_upper_long_s", "inputBase64": "mqNhL2KlYXNzZXShMavFv2hhMjU2OmFiYwCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMavFv2hhMjU2OmFiYwCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMavFv2hhMjU2OmFiYwCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:\u017Fha256:abc" }, { @@ -1656,14 +1684,14 @@ "label": "array_15", "inputBase64": "n6NhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdsDAwMDA", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "metadata_15_entries", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiPomswoXaiazGhdqJrMqF2omszoXaiazShdqJrNaF2oms2oXaiazehdqJrOKF2oms5oXajazEwoXajazExoXajazEyoXajazEzoXajazE0oXY=", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiPomswoXaiazGhdqJrMqF2omszoXaiazShdqJrNaF2oms2oXaiazehdqJrOKF2oms5oXajazEwoXajazExoXajazEyoXajazEzoXajazE0oXY=", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4j6JrMKF2omsxoXaiazKhdqJrM6F2oms0oXaiazWhdqJrNqF2oms3oXaiazihdqJrOaF2o2sxMKF2o2sxMaF2o2sxMqF2o2sxM6F2o2sxNKF2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { @@ -1698,147 +1726,147 @@ "label": "width_u8_200", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NczIr2FwcGxpY2F0aW9uL3gtdKF0kZTZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAzMiha5GheIGha6F2", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NczIr2FwcGxpY2F0aW9uL3gtdKF0kZTZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAzMiha5GheIGha6F2", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NczIr2FwcGxpY2F0aW9uL3gtdKF0kZXZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAzMiha6CRoXiBoWuhdg==", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "width_u8_200_offset", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc0Aya9hcHBsaWNhdGlvbi94LXShdJKU2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AM0AyKFrlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NczIAaFrkaF4gaFroXY=", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NczJr2FwcGxpY2F0aW9uL3gtdKF0kpTZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAzMiha5TZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTXMyAGha5GheIGha6F2", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NczJr2FwcGxpY2F0aW9uL3gtdKF0kpXZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAzMiha6CV2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1zMgBoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "width_u16_258", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc0BAq9hcHBsaWNhdGlvbi94LXShdJGU2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AM0BAqFrkaF4gaFroXY=", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc0BAq9hcHBsaWNhdGlvbi94LXShdJGU2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AM0BAqFrkaF4gaFroXY=", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc0BAq9hcHBsaWNhdGlvbi94LXShdJGV2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AM0BAqFroJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "width_u16_258_offset", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc0BA69hcHBsaWNhdGlvbi94LXShdJKU2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AM0BAqFrlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc0BAgGha5GheIGha6F2", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc0BA69hcHBsaWNhdGlvbi94LXShdJKU2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AM0BAqFrlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc0BAgGha5GheIGha6F2", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc0BA69hcHBsaWNhdGlvbi94LXShdJKV2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AM0BAqFroJXZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTXNAQIBoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "width_u16_40000", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc2cQK9hcHBsaWNhdGlvbi94LXShdJGU2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AM2cQKFrkaF4gaFroXY=", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc2cQK9hcHBsaWNhdGlvbi94LXShdJGU2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AM2cQKFrkaF4gaFroXY=", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc2cQK9hcHBsaWNhdGlvbi94LXShdJGV2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AM2cQKFroJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "width_u16_40000_offset", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc2cQa9hcHBsaWNhdGlvbi94LXShdJKU2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AM2cQKFrlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc2cQAGha5GheIGha6F2", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc2cQa9hcHBsaWNhdGlvbi94LXShdJKU2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AM2cQKFrlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc2cQAGha5GheIGha6F2", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc2cQa9hcHBsaWNhdGlvbi94LXShdJKV2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AM2cQKFroJXZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTXNnEABoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "width_u32_258", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc4AAAECr2FwcGxpY2F0aW9uL3gtdKF0kZTZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAzgAAAQKha5GheIGha6F2", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc0BAq9hcHBsaWNhdGlvbi94LXShdJGU2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AM0BAqFrkaF4gaFroXY=", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc0BAq9hcHBsaWNhdGlvbi94LXShdJGV2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AM0BAqFroJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "width_u32_258_offset", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc0BA69hcHBsaWNhdGlvbi94LXShdJKU2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AM0BAqFrlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc4AAAECAaFrkaF4gaFroXY=", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc0BA69hcHBsaWNhdGlvbi94LXShdJKU2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AM0BAqFrlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc0BAgGha5GheIGha6F2", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc0BA69hcHBsaWNhdGlvbi94LXShdJKV2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AM0BAqFroJXZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTXNAQIBoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "width_u64_258", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc8AAAAAAAABAq9hcHBsaWNhdGlvbi94LXShdJGU2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AM8AAAAAAAABAqFrkaF4gaFroXY=", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc0BAq9hcHBsaWNhdGlvbi94LXShdJGU2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AM0BAqFrkaF4gaFroXY=", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc0BAq9hcHBsaWNhdGlvbi94LXShdJGV2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AM0BAqFroJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "width_u64_258_offset", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc0BA69hcHBsaWNhdGlvbi94LXShdJKU2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AM0BAqFrlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc8AAAAAAAABAgGha5GheIGha6F2", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc0BA69hcHBsaWNhdGlvbi94LXShdJKU2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AM0BAqFrlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc0BAgGha5GheIGha6F2", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc0BA69hcHBsaWNhdGlvbi94LXShdJKV2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AM0BAqFroJXZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTXNAQIBoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "width_i8_100", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NdBkr2FwcGxpY2F0aW9uL3gtdKF0kZTZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUA0GSha5GheIGha6F2", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NWSvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQBkoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NWSvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQBkoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "width_i8_100_offset", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc0AZa9hcHBsaWNhdGlvbi94LXShdJKU2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AM0AZKFrlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NdBkAaFrkaF4gaFroXY=", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NWWvYXBwbGljYXRpb24veC10oXSSlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQBkoWuU2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1ZAGha5GheIGha6F2", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NWWvYXBwbGljYXRpb24veC10oXSSldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQBkoWugldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NWQBoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "width_i16_258", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NdEBAq9hcHBsaWNhdGlvbi94LXShdJGU2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1ANEBAqFrkaF4gaFroXY=", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc0BAq9hcHBsaWNhdGlvbi94LXShdJGU2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AM0BAqFrkaF4gaFroXY=", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc0BAq9hcHBsaWNhdGlvbi94LXShdJGV2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AM0BAqFroJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "width_i16_258_offset", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc0BA69hcHBsaWNhdGlvbi94LXShdJKU2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AM0BAqFrlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NdEBAgGha5GheIGha6F2", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc0BA69hcHBsaWNhdGlvbi94LXShdJKU2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AM0BAqFrlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc0BAgGha5GheIGha6F2", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc0BA69hcHBsaWNhdGlvbi94LXShdJKV2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AM0BAqFroJXZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTXNAQIBoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "width_i32_258", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NdIAAAECr2FwcGxpY2F0aW9uL3gtdKF0kZTZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUA0gAAAQKha5GheIGha6F2", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc0BAq9hcHBsaWNhdGlvbi94LXShdJGU2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AM0BAqFrkaF4gaFroXY=", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc0BAq9hcHBsaWNhdGlvbi94LXShdJGV2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AM0BAqFroJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "width_i32_258_offset", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc0BA69hcHBsaWNhdGlvbi94LXShdJKU2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AM0BAqFrlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NdIAAAECAaFrkaF4gaFroXY=", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc0BA69hcHBsaWNhdGlvbi94LXShdJKU2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AM0BAqFrlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc0BAgGha5GheIGha6F2", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc0BA69hcHBsaWNhdGlvbi94LXShdJKV2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AM0BAqFroJXZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTXNAQIBoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "width_i64_258", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NdMAAAAAAAABAq9hcHBsaWNhdGlvbi94LXShdJGU2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1ANMAAAAAAAABAqFrkaF4gaFroXY=", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc0BAq9hcHBsaWNhdGlvbi94LXShdJGU2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AM0BAqFrkaF4gaFroXY=", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc0BAq9hcHBsaWNhdGlvbi94LXShdJGV2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AM0BAqFroJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "width_i64_258_offset", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc0BA69hcHBsaWNhdGlvbi94LXShdJKU2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AM0BAqFrlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NdMAAAAAAAABAgGha5GheIGha6F2", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc0BA69hcHBsaWNhdGlvbi94LXShdJKU2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AM0BAqFrlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc0BAgGha5GheIGha6F2", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc0BA69hcHBsaWNhdGlvbi94LXShdJKV2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AM0BAqFroJXZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTXNAQIBoWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "width_fixint_127", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NX\u002BvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQB/oWuRoXiBoWuhdg==", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NX\u002BvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQB/oWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NX\u002BvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQB/oWugkaF4gaFroXY=", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "width_fixint_127_offset", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc0AgK9hcHBsaWNhdGlvbi94LXShdJKU2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AM0Af6FrlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NX8BoWuRoXiBoWuhdg==", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NcyAr2FwcGxpY2F0aW9uL3gtdKF0kpTZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAf6FrlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NX8BoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NcyAr2FwcGxpY2F0aW9uL3gtdKF0kpXZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAf6FroJXZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTV/AaFroJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "label": "width_u32_over_int32", "inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc6y0F4Ar2FwcGxpY2F0aW9uL3gtdKF0kpTZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAzn////\u002Bha5TZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTXOf////84y0F4BoWuRoXiBoWuhdg==", "accepted": true, - "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc6y0F4Ar2FwcGxpY2F0aW9uL3gtdKF0kpTZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAzn////\u002Bha5TZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTXOf////84y0F4BoWuRoXiBoWuhdg==", + "reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1Nc6y0F4Ar2FwcGxpY2F0aW9uL3gtdKF0kpXZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAzn////\u002Bha6CV2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1zn/////OMtBeAaFroJGheIGha6F2", "recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { diff --git a/contracts/cultmesh/content-vectors.cs-written.json b/contracts/cultmesh/content-vectors.cs-written.json index 0eaaaece..26444761 100644 --- a/contracts/cultmesh/content-vectors.cs-written.json +++ b/contracts/cultmesh/content-vectors.cs-written.json @@ -168,22 +168,22 @@ { "label": "manifest", "kind": "manifest", - "messagePackBase64": "mrVldXJla2EvY29udGVudC12ZWN0b3KlYXNzZXShMdlAZjRhZTJhN2M5NWY0ZDk2NjA1OGY3NWMxMzE4NWFhMGFiNmE5MWZjNWFmYWQ4ODYyOTZlZjJhZWU3NTVhMmFmMc4AFBLfvGFwcGxpY2F0aW9uL3gtY3VsdG5ldC12ZWN0b3LZITIwMjYtMDktMjJUMDA6MDA6MDAuMDAwMDAwMCswMDowMJaU2UAzMWExZjlkZWEwMTY5NTUxMDkyZDA1ZThiZjRhNDQ2MjI4YzhjM2ViNGM5YjcxM2M2NmFkY2I3ZmQ1M2M4OWJlAM4ABAAA2U9tZXNoOmNkbjpjaHVuazozMWExZjlkZWEwMTY5NTUxMDkyZDA1ZThiZjRhNDQ2MjI4YzhjM2ViNGM5YjcxM2M2NmFkY2I3ZmQ1M2M4OWJllNlAZTMxYmRjNjM1NzdmZTc2ZjY0NDEwMTRhYWFhMjIyNmQ5ZmZjZTExYjQ0ZDBkNTEwY2FkZTEzMmI3OWZkN2Q3Ns4ABAAAzgAEAADZT21lc2g6Y2RuOmNodW5rOmUzMWJkYzYzNTc3ZmU3NmY2NDQxMDE0YWFhYTIyMjZkOWZmY2UxMWI0NGQwZDUxMGNhZGUxMzJiNzlmZDdkNzaU2UBjN2YxYmRlOWUxZWZhN2QxMzg1ZDUxYjA3N2VhODdiMGMzODYwZDI4ZGE0M2Q1ZDZmMjIxZTg0NWM0YzIzNjBmzgAIAADOAAQAANlPbWVzaDpjZG46Y2h1bms6YzdmMWJkZTllMWVmYTdkMTM4NWQ1MWIwNzdlYTg3YjBjMzg2MGQyOGRhNDNkNWQ2ZjIyMWU4NDVjNGMyMzYwZpTZQGFhMTQ0NGJlNGI2NDA5OGY1ZGQ4NjNiNWIwYWM1NjE0ODBiM2ZiNWM0MWZjMDdjZjUwYTYyZWRjOTQxMjRiY2POAAwAAM4ABAAA2U9tZXNoOmNkbjpjaHVuazphYTE0NDRiZTRiNjQwOThmNWRkODYzYjViMGFjNTYxNDgwYjNmYjVjNDFmYzA3Y2Y1MGE2MmVkYzk0MTI0YmNjlNlAMjgzMTcxNTg1OWYwMWYzZjk2M2RkMGYxMTk0ZGJkYmI3NjA5OTk3NDIzMDE1ZDMxZjYwNzkxMWUxYmY4NjQwMs4AEAAAzgAEAADZT21lc2g6Y2RuOmNodW5rOjI4MzE3MTU4NTlmMDFmM2Y5NjNkZDBmMTE5NGRiZGJiNzYwOTk5NzQyMzAxNWQzMWY2MDc5MTFlMWJmODY0MDKU2UA1OTkzYWM2MDQwN2U4NjYwMTg3M2JjMzhlZTZmOGRiM2VhOTEyZTk4NzYxY2I1NThhYjk3OTVkYjMwNmI0NzRjzgAUAADNEt/ZT21lc2g6Y2RuOmNodW5rOjU5OTNhYzYwNDA3ZTg2NjAxODczYmMzOGVlNmY4ZGIzZWE5MTJlOTg3NjFjYjU1OGFiOTc5NWRiMzA2YjQ3NGOQgA==" + "messagePackBase64": "mrVldXJla2EvY29udGVudC12ZWN0b3KlYXNzZXShMdlAZjRhZTJhN2M5NWY0ZDk2NjA1OGY3NWMxMzE4NWFhMGFiNmE5MWZjNWFmYWQ4ODYyOTZlZjJhZWU3NTVhMmFmMc4AFBLfvGFwcGxpY2F0aW9uL3gtY3VsdG5ldC12ZWN0b3LZITIwMjYtMDktMjJUMDA6MDA6MDAuMDAwMDAwMCswMDowMJaV2UAzMWExZjlkZWEwMTY5NTUxMDkyZDA1ZThiZjRhNDQ2MjI4YzhjM2ViNGM5YjcxM2M2NmFkY2I3ZmQ1M2M4OWJlAM4ABAAA2U9tZXNoOmNkbjpjaHVuazozMWExZjlkZWEwMTY5NTUxMDkyZDA1ZThiZjRhNDQ2MjI4YzhjM2ViNGM5YjcxM2M2NmFkY2I3ZmQ1M2M4OWJloTCV2UBlMzFiZGM2MzU3N2ZlNzZmNjQ0MTAxNGFhYWEyMjI2ZDlmZmNlMTFiNDRkMGQ1MTBjYWRlMTMyYjc5ZmQ3ZDc2zgAEAADOAAQAANlPbWVzaDpjZG46Y2h1bms6ZTMxYmRjNjM1NzdmZTc2ZjY0NDEwMTRhYWFhMjIyNmQ5ZmZjZTExYjQ0ZDBkNTEwY2FkZTEzMmI3OWZkN2Q3NqYyNjIxNDSV2UBjN2YxYmRlOWUxZWZhN2QxMzg1ZDUxYjA3N2VhODdiMGMzODYwZDI4ZGE0M2Q1ZDZmMjIxZTg0NWM0YzIzNjBmzgAIAADOAAQAANlPbWVzaDpjZG46Y2h1bms6YzdmMWJkZTllMWVmYTdkMTM4NWQ1MWIwNzdlYTg3YjBjMzg2MGQyOGRhNDNkNWQ2ZjIyMWU4NDVjNGMyMzYwZqY1MjQyODiV2UBhYTE0NDRiZTRiNjQwOThmNWRkODYzYjViMGFjNTYxNDgwYjNmYjVjNDFmYzA3Y2Y1MGE2MmVkYzk0MTI0YmNjzgAMAADOAAQAANlPbWVzaDpjZG46Y2h1bms6YWExNDQ0YmU0YjY0MDk4ZjVkZDg2M2I1YjBhYzU2MTQ4MGIzZmI1YzQxZmMwN2NmNTBhNjJlZGM5NDEyNGJjY6Y3ODY0MzKV2UAyODMxNzE1ODU5ZjAxZjNmOTYzZGQwZjExOTRkYmRiYjc2MDk5OTc0MjMwMTVkMzFmNjA3OTExZTFiZjg2NDAyzgAQAADOAAQAANlPbWVzaDpjZG46Y2h1bms6MjgzMTcxNTg1OWYwMWYzZjk2M2RkMGYxMTk0ZGJkYmI3NjA5OTk3NDIzMDE1ZDMxZjYwNzkxMWUxYmY4NjQwMqcxMDQ4NTc2ldlANTk5M2FjNjA0MDdlODY2MDE4NzNiYzM4ZWU2ZjhkYjNlYTkxMmU5ODc2MWNiNTU4YWI5Nzk1ZGIzMDZiNDc0Y84AFAAAzRLf2U9tZXNoOmNkbjpjaHVuazo1OTkzYWM2MDQwN2U4NjYwMTg3M2JjMzhlZTZmOGRiM2VhOTEyZTk4NzYxY2I1NThhYjk3OTVkYjMwNmI0NzRjpzEzMTA3MjCQgA==" }, { "label": "manifest_out_of_order", "kind": "manifest", - "messagePackBase64": "mrVldXJla2EvY29udGVudC12ZWN0b3KlYXNzZXShMdlAZjRhZTJhN2M5NWY0ZDk2NjA1OGY3NWMxMzE4NWFhMGFiNmE5MWZjNWFmYWQ4ODYyOTZlZjJhZWU3NTVhMmFmMc4AFBLfvGFwcGxpY2F0aW9uL3gtY3VsdG5ldC12ZWN0b3LZITIwMjYtMDktMjJUMDA6MDA6MDAuMDAwMDAwMCswMDowMJaU2UBhYTE0NDRiZTRiNjQwOThmNWRkODYzYjViMGFjNTYxNDgwYjNmYjVjNDFmYzA3Y2Y1MGE2MmVkYzk0MTI0YmNjzgAMAADOAAQAANlPbWVzaDpjZG46Y2h1bms6YWExNDQ0YmU0YjY0MDk4ZjVkZDg2M2I1YjBhYzU2MTQ4MGIzZmI1YzQxZmMwN2NmNTBhNjJlZGM5NDEyNGJjY5TZQDMxYTFmOWRlYTAxNjk1NTEwOTJkMDVlOGJmNGE0NDYyMjhjOGMzZWI0YzliNzEzYzY2YWRjYjdmZDUzYzg5YmUAzgAEAADZT21lc2g6Y2RuOmNodW5rOjMxYTFmOWRlYTAxNjk1NTEwOTJkMDVlOGJmNGE0NDYyMjhjOGMzZWI0YzliNzEzYzY2YWRjYjdmZDUzYzg5YmWU2UA1OTkzYWM2MDQwN2U4NjYwMTg3M2JjMzhlZTZmOGRiM2VhOTEyZTk4NzYxY2I1NThhYjk3OTVkYjMwNmI0NzRjzgAUAADNEt/ZT21lc2g6Y2RuOmNodW5rOjU5OTNhYzYwNDA3ZTg2NjAxODczYmMzOGVlNmY4ZGIzZWE5MTJlOTg3NjFjYjU1OGFiOTc5NWRiMzA2YjQ3NGOU2UBlMzFiZGM2MzU3N2ZlNzZmNjQ0MTAxNGFhYWEyMjI2ZDlmZmNlMTFiNDRkMGQ1MTBjYWRlMTMyYjc5ZmQ3ZDc2zgAEAADOAAQAANlPbWVzaDpjZG46Y2h1bms6ZTMxYmRjNjM1NzdmZTc2ZjY0NDEwMTRhYWFhMjIyNmQ5ZmZjZTExYjQ0ZDBkNTEwY2FkZTEzMmI3OWZkN2Q3NpTZQDI4MzE3MTU4NTlmMDFmM2Y5NjNkZDBmMTE5NGRiZGJiNzYwOTk5NzQyMzAxNWQzMWY2MDc5MTFlMWJmODY0MDLOABAAAM4ABAAA2U9tZXNoOmNkbjpjaHVuazoyODMxNzE1ODU5ZjAxZjNmOTYzZGQwZjExOTRkYmRiYjc2MDk5OTc0MjMwMTVkMzFmNjA3OTExZTFiZjg2NDAylNlAYzdmMWJkZTllMWVmYTdkMTM4NWQ1MWIwNzdlYTg3YjBjMzg2MGQyOGRhNDNkNWQ2ZjIyMWU4NDVjNGMyMzYwZs4ACAAAzgAEAADZT21lc2g6Y2RuOmNodW5rOmM3ZjFiZGU5ZTFlZmE3ZDEzODVkNTFiMDc3ZWE4N2IwYzM4NjBkMjhkYTQzZDVkNmYyMjFlODQ1YzRjMjM2MGaQgA==" + "messagePackBase64": "mrVldXJla2EvY29udGVudC12ZWN0b3KlYXNzZXShMdlAZjRhZTJhN2M5NWY0ZDk2NjA1OGY3NWMxMzE4NWFhMGFiNmE5MWZjNWFmYWQ4ODYyOTZlZjJhZWU3NTVhMmFmMc4AFBLfvGFwcGxpY2F0aW9uL3gtY3VsdG5ldC12ZWN0b3LZITIwMjYtMDktMjJUMDA6MDA6MDAuMDAwMDAwMCswMDowMJaV2UBhYTE0NDRiZTRiNjQwOThmNWRkODYzYjViMGFjNTYxNDgwYjNmYjVjNDFmYzA3Y2Y1MGE2MmVkYzk0MTI0YmNjzgAMAADOAAQAANlPbWVzaDpjZG46Y2h1bms6YWExNDQ0YmU0YjY0MDk4ZjVkZDg2M2I1YjBhYzU2MTQ4MGIzZmI1YzQxZmMwN2NmNTBhNjJlZGM5NDEyNGJjY6Y3ODY0MzKV2UAzMWExZjlkZWEwMTY5NTUxMDkyZDA1ZThiZjRhNDQ2MjI4YzhjM2ViNGM5YjcxM2M2NmFkY2I3ZmQ1M2M4OWJlAM4ABAAA2U9tZXNoOmNkbjpjaHVuazozMWExZjlkZWEwMTY5NTUxMDkyZDA1ZThiZjRhNDQ2MjI4YzhjM2ViNGM5YjcxM2M2NmFkY2I3ZmQ1M2M4OWJloTCV2UA1OTkzYWM2MDQwN2U4NjYwMTg3M2JjMzhlZTZmOGRiM2VhOTEyZTk4NzYxY2I1NThhYjk3OTVkYjMwNmI0NzRjzgAUAADNEt/ZT21lc2g6Y2RuOmNodW5rOjU5OTNhYzYwNDA3ZTg2NjAxODczYmMzOGVlNmY4ZGIzZWE5MTJlOTg3NjFjYjU1OGFiOTc5NWRiMzA2YjQ3NGOnMTMxMDcyMJXZQGUzMWJkYzYzNTc3ZmU3NmY2NDQxMDE0YWFhYTIyMjZkOWZmY2UxMWI0NGQwZDUxMGNhZGUxMzJiNzlmZDdkNzbOAAQAAM4ABAAA2U9tZXNoOmNkbjpjaHVuazplMzFiZGM2MzU3N2ZlNzZmNjQ0MTAxNGFhYWEyMjI2ZDlmZmNlMTFiNDRkMGQ1MTBjYWRlMTMyYjc5ZmQ3ZDc2pjI2MjE0NJXZQDI4MzE3MTU4NTlmMDFmM2Y5NjNkZDBmMTE5NGRiZGJiNzYwOTk5NzQyMzAxNWQzMWY2MDc5MTFlMWJmODY0MDLOABAAAM4ABAAA2U9tZXNoOmNkbjpjaHVuazoyODMxNzE1ODU5ZjAxZjNmOTYzZGQwZjExOTRkYmRiYjc2MDk5OTc0MjMwMTVkMzFmNjA3OTExZTFiZjg2NDAypzEwNDg1NzaV2UBjN2YxYmRlOWUxZWZhN2QxMzg1ZDUxYjA3N2VhODdiMGMzODYwZDI4ZGE0M2Q1ZDZmMjIxZTg0NWM0YzIzNjBmzgAIAADOAAQAANlPbWVzaDpjZG46Y2h1bms6YzdmMWJkZTllMWVmYTdkMTM4NWQ1MWIwNzdlYTg3YjBjMzg2MGQyOGRhNDNkNWQ2ZjIyMWU4NDVjNGMyMzYwZqY1MjQyODiQgA==" }, { "label": "manifest_metadata_order", "kind": "manifest", - "messagePackBase64": "mrVldXJla2EvY29udGVudC12ZWN0b3KlYXNzZXShMdlAZjRhZTJhN2M5NWY0ZDk2NjA1OGY3NWMxMzE4NWFhMGFiNmE5MWZjNWFmYWQ4ODYyOTZlZjJhZWU3NTVhMmFmMc4AFBLfvGFwcGxpY2F0aW9uL3gtY3VsdG5ldC12ZWN0b3LZITIwMjYtMDktMjJUMDA6MDA6MDAuMDAwMDAwMCswMDowMJaU2UAzMWExZjlkZWEwMTY5NTUxMDkyZDA1ZThiZjRhNDQ2MjI4YzhjM2ViNGM5YjcxM2M2NmFkY2I3ZmQ1M2M4OWJlAM4ABAAA2U9tZXNoOmNkbjpjaHVuazozMWExZjlkZWEwMTY5NTUxMDkyZDA1ZThiZjRhNDQ2MjI4YzhjM2ViNGM5YjcxM2M2NmFkY2I3ZmQ1M2M4OWJllNlAZTMxYmRjNjM1NzdmZTc2ZjY0NDEwMTRhYWFhMjIyNmQ5ZmZjZTExYjQ0ZDBkNTEwY2FkZTEzMmI3OWZkN2Q3Ns4ABAAAzgAEAADZT21lc2g6Y2RuOmNodW5rOmUzMWJkYzYzNTc3ZmU3NmY2NDQxMDE0YWFhYTIyMjZkOWZmY2UxMWI0NGQwZDUxMGNhZGUxMzJiNzlmZDdkNzaU2UBjN2YxYmRlOWUxZWZhN2QxMzg1ZDUxYjA3N2VhODdiMGMzODYwZDI4ZGE0M2Q1ZDZmMjIxZTg0NWM0YzIzNjBmzgAIAADOAAQAANlPbWVzaDpjZG46Y2h1bms6YzdmMWJkZTllMWVmYTdkMTM4NWQ1MWIwNzdlYTg3YjBjMzg2MGQyOGRhNDNkNWQ2ZjIyMWU4NDVjNGMyMzYwZpTZQGFhMTQ0NGJlNGI2NDA5OGY1ZGQ4NjNiNWIwYWM1NjE0ODBiM2ZiNWM0MWZjMDdjZjUwYTYyZWRjOTQxMjRiY2POAAwAAM4ABAAA2U9tZXNoOmNkbjpjaHVuazphYTE0NDRiZTRiNjQwOThmNWRkODYzYjViMGFjNTYxNDgwYjNmYjVjNDFmYzA3Y2Y1MGE2MmVkYzk0MTI0YmNjlNlAMjgzMTcxNTg1OWYwMWYzZjk2M2RkMGYxMTk0ZGJkYmI3NjA5OTk3NDIzMDE1ZDMxZjYwNzkxMWUxYmY4NjQwMs4AEAAAzgAEAADZT21lc2g6Y2RuOmNodW5rOjI4MzE3MTU4NTlmMDFmM2Y5NjNkZDBmMTE5NGRiZGJiNzYwOTk5NzQyMzAxNWQzMWY2MDc5MTFlMWJmODY0MDKU2UA1OTkzYWM2MDQwN2U4NjYwMTg3M2JjMzhlZTZmOGRiM2VhOTEyZTk4NzYxY2I1NThhYjk3OTVkYjMwNmI0NzRjzgAUAADNEt/ZT21lc2g6Y2RuOmNodW5rOjU5OTNhYzYwNDA3ZTg2NjAxODczYmMzOGVlNmY4ZGIzZWE5MTJlOTg3NjFjYjU1OGFiOTc5NWRiMzA2YjQ3NGOQg6R6ZXRhoTGlYWxwaGGhMqNtaWShMw==" + "messagePackBase64": "mrVldXJla2EvY29udGVudC12ZWN0b3KlYXNzZXShMdlAZjRhZTJhN2M5NWY0ZDk2NjA1OGY3NWMxMzE4NWFhMGFiNmE5MWZjNWFmYWQ4ODYyOTZlZjJhZWU3NTVhMmFmMc4AFBLfvGFwcGxpY2F0aW9uL3gtY3VsdG5ldC12ZWN0b3LZITIwMjYtMDktMjJUMDA6MDA6MDAuMDAwMDAwMCswMDowMJaV2UAzMWExZjlkZWEwMTY5NTUxMDkyZDA1ZThiZjRhNDQ2MjI4YzhjM2ViNGM5YjcxM2M2NmFkY2I3ZmQ1M2M4OWJlAM4ABAAA2U9tZXNoOmNkbjpjaHVuazozMWExZjlkZWEwMTY5NTUxMDkyZDA1ZThiZjRhNDQ2MjI4YzhjM2ViNGM5YjcxM2M2NmFkY2I3ZmQ1M2M4OWJloTCV2UBlMzFiZGM2MzU3N2ZlNzZmNjQ0MTAxNGFhYWEyMjI2ZDlmZmNlMTFiNDRkMGQ1MTBjYWRlMTMyYjc5ZmQ3ZDc2zgAEAADOAAQAANlPbWVzaDpjZG46Y2h1bms6ZTMxYmRjNjM1NzdmZTc2ZjY0NDEwMTRhYWFhMjIyNmQ5ZmZjZTExYjQ0ZDBkNTEwY2FkZTEzMmI3OWZkN2Q3NqYyNjIxNDSV2UBjN2YxYmRlOWUxZWZhN2QxMzg1ZDUxYjA3N2VhODdiMGMzODYwZDI4ZGE0M2Q1ZDZmMjIxZTg0NWM0YzIzNjBmzgAIAADOAAQAANlPbWVzaDpjZG46Y2h1bms6YzdmMWJkZTllMWVmYTdkMTM4NWQ1MWIwNzdlYTg3YjBjMzg2MGQyOGRhNDNkNWQ2ZjIyMWU4NDVjNGMyMzYwZqY1MjQyODiV2UBhYTE0NDRiZTRiNjQwOThmNWRkODYzYjViMGFjNTYxNDgwYjNmYjVjNDFmYzA3Y2Y1MGE2MmVkYzk0MTI0YmNjzgAMAADOAAQAANlPbWVzaDpjZG46Y2h1bms6YWExNDQ0YmU0YjY0MDk4ZjVkZDg2M2I1YjBhYzU2MTQ4MGIzZmI1YzQxZmMwN2NmNTBhNjJlZGM5NDEyNGJjY6Y3ODY0MzKV2UAyODMxNzE1ODU5ZjAxZjNmOTYzZGQwZjExOTRkYmRiYjc2MDk5OTc0MjMwMTVkMzFmNjA3OTExZTFiZjg2NDAyzgAQAADOAAQAANlPbWVzaDpjZG46Y2h1bms6MjgzMTcxNTg1OWYwMWYzZjk2M2RkMGYxMTk0ZGJkYmI3NjA5OTk3NDIzMDE1ZDMxZjYwNzkxMWUxYmY4NjQwMqcxMDQ4NTc2ldlANTk5M2FjNjA0MDdlODY2MDE4NzNiYzM4ZWU2ZjhkYjNlYTkxMmU5ODc2MWNiNTU4YWI5Nzk1ZGIzMDZiNDc0Y84AFAAAzRLf2U9tZXNoOmNkbjpjaHVuazo1OTkzYWM2MDQwN2U4NjYwMTg3M2JjMzhlZTZmOGRiM2VhOTEyZTk4NzYxY2I1NThhYjk3OTVkYjMwNmI0NzRjpzEzMTA3MjCQg6R6ZXRhoTGlYWxwaGGhMqNtaWShMw==" }, { "label": "manifest_empty_body", "kind": "manifest", - "messagePackBase64": "mrVldXJla2EvY29udGVudC12ZWN0b3KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQC8YXBwbGljYXRpb24veC1jdWx0bmV0LXZlY3RvctkhMjAyNi0wOS0yMlQwMDowMDowMC4wMDAwMDAwKzAwOjAwkZTZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAANlPbWVzaDpjZG46Y2h1bms6ZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NZCA" + "messagePackBase64": "mrVldXJla2EvY29udGVudC12ZWN0b3KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQC8YXBwbGljYXRpb24veC1jdWx0bmV0LXZlY3RvctkhMjAyNi0wOS0yMlQwMDowMDowMC4wMDAwMDAwKzAwOjAwkZXZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAANlPbWVzaDpjZG46Y2h1bms6ZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NaEwkIA=" } ] -} +} \ No newline at end of file diff --git a/contracts/cultmesh/content-vectors.rs-written.json b/contracts/cultmesh/content-vectors.rs-written.json index 38538e2b..fbbe51ea 100644 --- a/contracts/cultmesh/content-vectors.rs-written.json +++ b/contracts/cultmesh/content-vectors.rs-written.json @@ -138,22 +138,22 @@ { "label": "manifest", "kind": "manifest", - "messagePackBase64": "mrVldXJla2EvY29udGVudC12ZWN0b3KlYXNzZXShMdlAZjRhZTJhN2M5NWY0ZDk2NjA1OGY3NWMxMzE4NWFhMGFiNmE5MWZjNWFmYWQ4ODYyOTZlZjJhZWU3NTVhMmFmMc4AFBLfvGFwcGxpY2F0aW9uL3gtY3VsdG5ldC12ZWN0b3LZITIwMjYtMDktMjJUMDA6MDA6MDAuMDAwMDAwMCswMDowMJaU2UAzMWExZjlkZWEwMTY5NTUxMDkyZDA1ZThiZjRhNDQ2MjI4YzhjM2ViNGM5YjcxM2M2NmFkY2I3ZmQ1M2M4OWJlAM4ABAAA2U9tZXNoOmNkbjpjaHVuazozMWExZjlkZWEwMTY5NTUxMDkyZDA1ZThiZjRhNDQ2MjI4YzhjM2ViNGM5YjcxM2M2NmFkY2I3ZmQ1M2M4OWJllNlAZTMxYmRjNjM1NzdmZTc2ZjY0NDEwMTRhYWFhMjIyNmQ5ZmZjZTExYjQ0ZDBkNTEwY2FkZTEzMmI3OWZkN2Q3Ns4ABAAAzgAEAADZT21lc2g6Y2RuOmNodW5rOmUzMWJkYzYzNTc3ZmU3NmY2NDQxMDE0YWFhYTIyMjZkOWZmY2UxMWI0NGQwZDUxMGNhZGUxMzJiNzlmZDdkNzaU2UBjN2YxYmRlOWUxZWZhN2QxMzg1ZDUxYjA3N2VhODdiMGMzODYwZDI4ZGE0M2Q1ZDZmMjIxZTg0NWM0YzIzNjBmzgAIAADOAAQAANlPbWVzaDpjZG46Y2h1bms6YzdmMWJkZTllMWVmYTdkMTM4NWQ1MWIwNzdlYTg3YjBjMzg2MGQyOGRhNDNkNWQ2ZjIyMWU4NDVjNGMyMzYwZpTZQGFhMTQ0NGJlNGI2NDA5OGY1ZGQ4NjNiNWIwYWM1NjE0ODBiM2ZiNWM0MWZjMDdjZjUwYTYyZWRjOTQxMjRiY2POAAwAAM4ABAAA2U9tZXNoOmNkbjpjaHVuazphYTE0NDRiZTRiNjQwOThmNWRkODYzYjViMGFjNTYxNDgwYjNmYjVjNDFmYzA3Y2Y1MGE2MmVkYzk0MTI0YmNjlNlAMjgzMTcxNTg1OWYwMWYzZjk2M2RkMGYxMTk0ZGJkYmI3NjA5OTk3NDIzMDE1ZDMxZjYwNzkxMWUxYmY4NjQwMs4AEAAAzgAEAADZT21lc2g6Y2RuOmNodW5rOjI4MzE3MTU4NTlmMDFmM2Y5NjNkZDBmMTE5NGRiZGJiNzYwOTk5NzQyMzAxNWQzMWY2MDc5MTFlMWJmODY0MDKU2UA1OTkzYWM2MDQwN2U4NjYwMTg3M2JjMzhlZTZmOGRiM2VhOTEyZTk4NzYxY2I1NThhYjk3OTVkYjMwNmI0NzRjzgAUAADNEt/ZT21lc2g6Y2RuOmNodW5rOjU5OTNhYzYwNDA3ZTg2NjAxODczYmMzOGVlNmY4ZGIzZWE5MTJlOTg3NjFjYjU1OGFiOTc5NWRiMzA2YjQ3NGOQgA==" + "messagePackBase64": "mrVldXJla2EvY29udGVudC12ZWN0b3KlYXNzZXShMdlAZjRhZTJhN2M5NWY0ZDk2NjA1OGY3NWMxMzE4NWFhMGFiNmE5MWZjNWFmYWQ4ODYyOTZlZjJhZWU3NTVhMmFmMc4AFBLfvGFwcGxpY2F0aW9uL3gtY3VsdG5ldC12ZWN0b3LZITIwMjYtMDktMjJUMDA6MDA6MDAuMDAwMDAwMCswMDowMJaV2UAzMWExZjlkZWEwMTY5NTUxMDkyZDA1ZThiZjRhNDQ2MjI4YzhjM2ViNGM5YjcxM2M2NmFkY2I3ZmQ1M2M4OWJlAM4ABAAA2U9tZXNoOmNkbjpjaHVuazozMWExZjlkZWEwMTY5NTUxMDkyZDA1ZThiZjRhNDQ2MjI4YzhjM2ViNGM5YjcxM2M2NmFkY2I3ZmQ1M2M4OWJloTCV2UBlMzFiZGM2MzU3N2ZlNzZmNjQ0MTAxNGFhYWEyMjI2ZDlmZmNlMTFiNDRkMGQ1MTBjYWRlMTMyYjc5ZmQ3ZDc2zgAEAADOAAQAANlPbWVzaDpjZG46Y2h1bms6ZTMxYmRjNjM1NzdmZTc2ZjY0NDEwMTRhYWFhMjIyNmQ5ZmZjZTExYjQ0ZDBkNTEwY2FkZTEzMmI3OWZkN2Q3NqYyNjIxNDSV2UBjN2YxYmRlOWUxZWZhN2QxMzg1ZDUxYjA3N2VhODdiMGMzODYwZDI4ZGE0M2Q1ZDZmMjIxZTg0NWM0YzIzNjBmzgAIAADOAAQAANlPbWVzaDpjZG46Y2h1bms6YzdmMWJkZTllMWVmYTdkMTM4NWQ1MWIwNzdlYTg3YjBjMzg2MGQyOGRhNDNkNWQ2ZjIyMWU4NDVjNGMyMzYwZqY1MjQyODiV2UBhYTE0NDRiZTRiNjQwOThmNWRkODYzYjViMGFjNTYxNDgwYjNmYjVjNDFmYzA3Y2Y1MGE2MmVkYzk0MTI0YmNjzgAMAADOAAQAANlPbWVzaDpjZG46Y2h1bms6YWExNDQ0YmU0YjY0MDk4ZjVkZDg2M2I1YjBhYzU2MTQ4MGIzZmI1YzQxZmMwN2NmNTBhNjJlZGM5NDEyNGJjY6Y3ODY0MzKV2UAyODMxNzE1ODU5ZjAxZjNmOTYzZGQwZjExOTRkYmRiYjc2MDk5OTc0MjMwMTVkMzFmNjA3OTExZTFiZjg2NDAyzgAQAADOAAQAANlPbWVzaDpjZG46Y2h1bms6MjgzMTcxNTg1OWYwMWYzZjk2M2RkMGYxMTk0ZGJkYmI3NjA5OTk3NDIzMDE1ZDMxZjYwNzkxMWUxYmY4NjQwMqcxMDQ4NTc2ldlANTk5M2FjNjA0MDdlODY2MDE4NzNiYzM4ZWU2ZjhkYjNlYTkxMmU5ODc2MWNiNTU4YWI5Nzk1ZGIzMDZiNDc0Y84AFAAAzRLf2U9tZXNoOmNkbjpjaHVuazo1OTkzYWM2MDQwN2U4NjYwMTg3M2JjMzhlZTZmOGRiM2VhOTEyZTk4NzYxY2I1NThhYjk3OTVkYjMwNmI0NzRjpzEzMTA3MjCQgA==" }, { "label": "manifest_out_of_order", "kind": "manifest", - "messagePackBase64": "mrVldXJla2EvY29udGVudC12ZWN0b3KlYXNzZXShMdlAZjRhZTJhN2M5NWY0ZDk2NjA1OGY3NWMxMzE4NWFhMGFiNmE5MWZjNWFmYWQ4ODYyOTZlZjJhZWU3NTVhMmFmMc4AFBLfvGFwcGxpY2F0aW9uL3gtY3VsdG5ldC12ZWN0b3LZITIwMjYtMDktMjJUMDA6MDA6MDAuMDAwMDAwMCswMDowMJaU2UBhYTE0NDRiZTRiNjQwOThmNWRkODYzYjViMGFjNTYxNDgwYjNmYjVjNDFmYzA3Y2Y1MGE2MmVkYzk0MTI0YmNjzgAMAADOAAQAANlPbWVzaDpjZG46Y2h1bms6YWExNDQ0YmU0YjY0MDk4ZjVkZDg2M2I1YjBhYzU2MTQ4MGIzZmI1YzQxZmMwN2NmNTBhNjJlZGM5NDEyNGJjY5TZQDMxYTFmOWRlYTAxNjk1NTEwOTJkMDVlOGJmNGE0NDYyMjhjOGMzZWI0YzliNzEzYzY2YWRjYjdmZDUzYzg5YmUAzgAEAADZT21lc2g6Y2RuOmNodW5rOjMxYTFmOWRlYTAxNjk1NTEwOTJkMDVlOGJmNGE0NDYyMjhjOGMzZWI0YzliNzEzYzY2YWRjYjdmZDUzYzg5YmWU2UA1OTkzYWM2MDQwN2U4NjYwMTg3M2JjMzhlZTZmOGRiM2VhOTEyZTk4NzYxY2I1NThhYjk3OTVkYjMwNmI0NzRjzgAUAADNEt/ZT21lc2g6Y2RuOmNodW5rOjU5OTNhYzYwNDA3ZTg2NjAxODczYmMzOGVlNmY4ZGIzZWE5MTJlOTg3NjFjYjU1OGFiOTc5NWRiMzA2YjQ3NGOU2UBlMzFiZGM2MzU3N2ZlNzZmNjQ0MTAxNGFhYWEyMjI2ZDlmZmNlMTFiNDRkMGQ1MTBjYWRlMTMyYjc5ZmQ3ZDc2zgAEAADOAAQAANlPbWVzaDpjZG46Y2h1bms6ZTMxYmRjNjM1NzdmZTc2ZjY0NDEwMTRhYWFhMjIyNmQ5ZmZjZTExYjQ0ZDBkNTEwY2FkZTEzMmI3OWZkN2Q3NpTZQDI4MzE3MTU4NTlmMDFmM2Y5NjNkZDBmMTE5NGRiZGJiNzYwOTk5NzQyMzAxNWQzMWY2MDc5MTFlMWJmODY0MDLOABAAAM4ABAAA2U9tZXNoOmNkbjpjaHVuazoyODMxNzE1ODU5ZjAxZjNmOTYzZGQwZjExOTRkYmRiYjc2MDk5OTc0MjMwMTVkMzFmNjA3OTExZTFiZjg2NDAylNlAYzdmMWJkZTllMWVmYTdkMTM4NWQ1MWIwNzdlYTg3YjBjMzg2MGQyOGRhNDNkNWQ2ZjIyMWU4NDVjNGMyMzYwZs4ACAAAzgAEAADZT21lc2g6Y2RuOmNodW5rOmM3ZjFiZGU5ZTFlZmE3ZDEzODVkNTFiMDc3ZWE4N2IwYzM4NjBkMjhkYTQzZDVkNmYyMjFlODQ1YzRjMjM2MGaQgA==" + "messagePackBase64": "mrVldXJla2EvY29udGVudC12ZWN0b3KlYXNzZXShMdlAZjRhZTJhN2M5NWY0ZDk2NjA1OGY3NWMxMzE4NWFhMGFiNmE5MWZjNWFmYWQ4ODYyOTZlZjJhZWU3NTVhMmFmMc4AFBLfvGFwcGxpY2F0aW9uL3gtY3VsdG5ldC12ZWN0b3LZITIwMjYtMDktMjJUMDA6MDA6MDAuMDAwMDAwMCswMDowMJaV2UBhYTE0NDRiZTRiNjQwOThmNWRkODYzYjViMGFjNTYxNDgwYjNmYjVjNDFmYzA3Y2Y1MGE2MmVkYzk0MTI0YmNjzgAMAADOAAQAANlPbWVzaDpjZG46Y2h1bms6YWExNDQ0YmU0YjY0MDk4ZjVkZDg2M2I1YjBhYzU2MTQ4MGIzZmI1YzQxZmMwN2NmNTBhNjJlZGM5NDEyNGJjY6Y3ODY0MzKV2UAzMWExZjlkZWEwMTY5NTUxMDkyZDA1ZThiZjRhNDQ2MjI4YzhjM2ViNGM5YjcxM2M2NmFkY2I3ZmQ1M2M4OWJlAM4ABAAA2U9tZXNoOmNkbjpjaHVuazozMWExZjlkZWEwMTY5NTUxMDkyZDA1ZThiZjRhNDQ2MjI4YzhjM2ViNGM5YjcxM2M2NmFkY2I3ZmQ1M2M4OWJloTCV2UA1OTkzYWM2MDQwN2U4NjYwMTg3M2JjMzhlZTZmOGRiM2VhOTEyZTk4NzYxY2I1NThhYjk3OTVkYjMwNmI0NzRjzgAUAADNEt/ZT21lc2g6Y2RuOmNodW5rOjU5OTNhYzYwNDA3ZTg2NjAxODczYmMzOGVlNmY4ZGIzZWE5MTJlOTg3NjFjYjU1OGFiOTc5NWRiMzA2YjQ3NGOnMTMxMDcyMJXZQGUzMWJkYzYzNTc3ZmU3NmY2NDQxMDE0YWFhYTIyMjZkOWZmY2UxMWI0NGQwZDUxMGNhZGUxMzJiNzlmZDdkNzbOAAQAAM4ABAAA2U9tZXNoOmNkbjpjaHVuazplMzFiZGM2MzU3N2ZlNzZmNjQ0MTAxNGFhYWEyMjI2ZDlmZmNlMTFiNDRkMGQ1MTBjYWRlMTMyYjc5ZmQ3ZDc2pjI2MjE0NJXZQDI4MzE3MTU4NTlmMDFmM2Y5NjNkZDBmMTE5NGRiZGJiNzYwOTk5NzQyMzAxNWQzMWY2MDc5MTFlMWJmODY0MDLOABAAAM4ABAAA2U9tZXNoOmNkbjpjaHVuazoyODMxNzE1ODU5ZjAxZjNmOTYzZGQwZjExOTRkYmRiYjc2MDk5OTc0MjMwMTVkMzFmNjA3OTExZTFiZjg2NDAypzEwNDg1NzaV2UBjN2YxYmRlOWUxZWZhN2QxMzg1ZDUxYjA3N2VhODdiMGMzODYwZDI4ZGE0M2Q1ZDZmMjIxZTg0NWM0YzIzNjBmzgAIAADOAAQAANlPbWVzaDpjZG46Y2h1bms6YzdmMWJkZTllMWVmYTdkMTM4NWQ1MWIwNzdlYTg3YjBjMzg2MGQyOGRhNDNkNWQ2ZjIyMWU4NDVjNGMyMzYwZqY1MjQyODiQgA==" }, { "label": "manifest_metadata_order", "kind": "manifest", - "messagePackBase64": "mrVldXJla2EvY29udGVudC12ZWN0b3KlYXNzZXShMdlAZjRhZTJhN2M5NWY0ZDk2NjA1OGY3NWMxMzE4NWFhMGFiNmE5MWZjNWFmYWQ4ODYyOTZlZjJhZWU3NTVhMmFmMc4AFBLfvGFwcGxpY2F0aW9uL3gtY3VsdG5ldC12ZWN0b3LZITIwMjYtMDktMjJUMDA6MDA6MDAuMDAwMDAwMCswMDowMJaU2UAzMWExZjlkZWEwMTY5NTUxMDkyZDA1ZThiZjRhNDQ2MjI4YzhjM2ViNGM5YjcxM2M2NmFkY2I3ZmQ1M2M4OWJlAM4ABAAA2U9tZXNoOmNkbjpjaHVuazozMWExZjlkZWEwMTY5NTUxMDkyZDA1ZThiZjRhNDQ2MjI4YzhjM2ViNGM5YjcxM2M2NmFkY2I3ZmQ1M2M4OWJllNlAZTMxYmRjNjM1NzdmZTc2ZjY0NDEwMTRhYWFhMjIyNmQ5ZmZjZTExYjQ0ZDBkNTEwY2FkZTEzMmI3OWZkN2Q3Ns4ABAAAzgAEAADZT21lc2g6Y2RuOmNodW5rOmUzMWJkYzYzNTc3ZmU3NmY2NDQxMDE0YWFhYTIyMjZkOWZmY2UxMWI0NGQwZDUxMGNhZGUxMzJiNzlmZDdkNzaU2UBjN2YxYmRlOWUxZWZhN2QxMzg1ZDUxYjA3N2VhODdiMGMzODYwZDI4ZGE0M2Q1ZDZmMjIxZTg0NWM0YzIzNjBmzgAIAADOAAQAANlPbWVzaDpjZG46Y2h1bms6YzdmMWJkZTllMWVmYTdkMTM4NWQ1MWIwNzdlYTg3YjBjMzg2MGQyOGRhNDNkNWQ2ZjIyMWU4NDVjNGMyMzYwZpTZQGFhMTQ0NGJlNGI2NDA5OGY1ZGQ4NjNiNWIwYWM1NjE0ODBiM2ZiNWM0MWZjMDdjZjUwYTYyZWRjOTQxMjRiY2POAAwAAM4ABAAA2U9tZXNoOmNkbjpjaHVuazphYTE0NDRiZTRiNjQwOThmNWRkODYzYjViMGFjNTYxNDgwYjNmYjVjNDFmYzA3Y2Y1MGE2MmVkYzk0MTI0YmNjlNlAMjgzMTcxNTg1OWYwMWYzZjk2M2RkMGYxMTk0ZGJkYmI3NjA5OTk3NDIzMDE1ZDMxZjYwNzkxMWUxYmY4NjQwMs4AEAAAzgAEAADZT21lc2g6Y2RuOmNodW5rOjI4MzE3MTU4NTlmMDFmM2Y5NjNkZDBmMTE5NGRiZGJiNzYwOTk5NzQyMzAxNWQzMWY2MDc5MTFlMWJmODY0MDKU2UA1OTkzYWM2MDQwN2U4NjYwMTg3M2JjMzhlZTZmOGRiM2VhOTEyZTk4NzYxY2I1NThhYjk3OTVkYjMwNmI0NzRjzgAUAADNEt/ZT21lc2g6Y2RuOmNodW5rOjU5OTNhYzYwNDA3ZTg2NjAxODczYmMzOGVlNmY4ZGIzZWE5MTJlOTg3NjFjYjU1OGFiOTc5NWRiMzA2YjQ3NGOQg6R6ZXRhoTGlYWxwaGGhMqNtaWShMw==" + "messagePackBase64": "mrVldXJla2EvY29udGVudC12ZWN0b3KlYXNzZXShMdlAZjRhZTJhN2M5NWY0ZDk2NjA1OGY3NWMxMzE4NWFhMGFiNmE5MWZjNWFmYWQ4ODYyOTZlZjJhZWU3NTVhMmFmMc4AFBLfvGFwcGxpY2F0aW9uL3gtY3VsdG5ldC12ZWN0b3LZITIwMjYtMDktMjJUMDA6MDA6MDAuMDAwMDAwMCswMDowMJaV2UAzMWExZjlkZWEwMTY5NTUxMDkyZDA1ZThiZjRhNDQ2MjI4YzhjM2ViNGM5YjcxM2M2NmFkY2I3ZmQ1M2M4OWJlAM4ABAAA2U9tZXNoOmNkbjpjaHVuazozMWExZjlkZWEwMTY5NTUxMDkyZDA1ZThiZjRhNDQ2MjI4YzhjM2ViNGM5YjcxM2M2NmFkY2I3ZmQ1M2M4OWJloTCV2UBlMzFiZGM2MzU3N2ZlNzZmNjQ0MTAxNGFhYWEyMjI2ZDlmZmNlMTFiNDRkMGQ1MTBjYWRlMTMyYjc5ZmQ3ZDc2zgAEAADOAAQAANlPbWVzaDpjZG46Y2h1bms6ZTMxYmRjNjM1NzdmZTc2ZjY0NDEwMTRhYWFhMjIyNmQ5ZmZjZTExYjQ0ZDBkNTEwY2FkZTEzMmI3OWZkN2Q3NqYyNjIxNDSV2UBjN2YxYmRlOWUxZWZhN2QxMzg1ZDUxYjA3N2VhODdiMGMzODYwZDI4ZGE0M2Q1ZDZmMjIxZTg0NWM0YzIzNjBmzgAIAADOAAQAANlPbWVzaDpjZG46Y2h1bms6YzdmMWJkZTllMWVmYTdkMTM4NWQ1MWIwNzdlYTg3YjBjMzg2MGQyOGRhNDNkNWQ2ZjIyMWU4NDVjNGMyMzYwZqY1MjQyODiV2UBhYTE0NDRiZTRiNjQwOThmNWRkODYzYjViMGFjNTYxNDgwYjNmYjVjNDFmYzA3Y2Y1MGE2MmVkYzk0MTI0YmNjzgAMAADOAAQAANlPbWVzaDpjZG46Y2h1bms6YWExNDQ0YmU0YjY0MDk4ZjVkZDg2M2I1YjBhYzU2MTQ4MGIzZmI1YzQxZmMwN2NmNTBhNjJlZGM5NDEyNGJjY6Y3ODY0MzKV2UAyODMxNzE1ODU5ZjAxZjNmOTYzZGQwZjExOTRkYmRiYjc2MDk5OTc0MjMwMTVkMzFmNjA3OTExZTFiZjg2NDAyzgAQAADOAAQAANlPbWVzaDpjZG46Y2h1bms6MjgzMTcxNTg1OWYwMWYzZjk2M2RkMGYxMTk0ZGJkYmI3NjA5OTk3NDIzMDE1ZDMxZjYwNzkxMWUxYmY4NjQwMqcxMDQ4NTc2ldlANTk5M2FjNjA0MDdlODY2MDE4NzNiYzM4ZWU2ZjhkYjNlYTkxMmU5ODc2MWNiNTU4YWI5Nzk1ZGIzMDZiNDc0Y84AFAAAzRLf2U9tZXNoOmNkbjpjaHVuazo1OTkzYWM2MDQwN2U4NjYwMTg3M2JjMzhlZTZmOGRiM2VhOTEyZTk4NzYxY2I1NThhYjk3OTVkYjMwNmI0NzRjpzEzMTA3MjCQg6R6ZXRhoTGlYWxwaGGhMqNtaWShMw==" }, { "label": "manifest_empty_body", "kind": "manifest", - "messagePackBase64": "mrVldXJla2EvY29udGVudC12ZWN0b3KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQC8YXBwbGljYXRpb24veC1jdWx0bmV0LXZlY3RvctkhMjAyNi0wOS0yMlQwMDowMDowMC4wMDAwMDAwKzAwOjAwkZTZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAANlPbWVzaDpjZG46Y2h1bms6ZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NZCA" + "messagePackBase64": "mrVldXJla2EvY29udGVudC12ZWN0b3KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQC8YXBwbGljYXRpb24veC1jdWx0bmV0LXZlY3RvctkhMjAyNi0wOS0yMlQwMDowMDowMC4wMDAwMDAwKzAwOjAwkZXZQGUzYjBjNDQyOThmYzFjMTQ5YWZiZjRjODk5NmZiOTI0MjdhZTQxZTQ2NDliOTM0Y2E0OTU5OTFiNzg1MmI4NTUAANlPbWVzaDpjZG46Y2h1bms6ZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NaEwkIA=" } ] -} +} \ No newline at end of file diff --git a/contracts/cultmesh/content-vectors.ts-written.json b/contracts/cultmesh/content-vectors.ts-written.json index be9e1549..4efd8cc3 100644 --- a/contracts/cultmesh/content-vectors.ts-written.json +++ b/contracts/cultmesh/content-vectors.ts-written.json @@ -35,7 +35,7 @@ } }, "recordKey": "mesh:cdn:artifact:pixel-heroes-hüt:unversioned:cf8a453a36981321977355448d24df49bcefee22be6e3730b9140e87c29013f9", - "messagePackBase64": "mrVQaXhlbCBIZXJvZXMvSMO8dCDimIOlYXNzZXSg2UBjZjhhNDUzYTM2OTgxMzIxOTc3MzU1NDQ4ZDI0ZGY0OWJjZWZlZTIyYmU2ZTM3MzBiOTE0MGU4N2MyOTAxM2Y5zgAL3jG/YXBwbGljYXRpb24veC1jdWx0bmV0LXRzLXZlY3RvctkhMjAyNi0xMC0xOVQwMDowMDowMC4wMDAwMDAwKzAwOjAwlpTZQDM3N2M3YjBjYzIyOWY0NzA1NjcwZDYzZDU0NGMyNzViNjYzZGY2ZThjODMxYjA5ODQ2YTVjNDhjY2EwOGI4NDgAzgACAADZT21lc2g6Y2RuOmNodW5rOjM3N2M3YjBjYzIyOWY0NzA1NjcwZDYzZDU0NGMyNzViNjYzZGY2ZThjODMxYjA5ODQ2YTVjNDhjY2EwOGI4NDiU2UAzNzdjN2IwY2MyMjlmNDcwNTY3MGQ2M2Q1NDRjMjc1YjY2M2RmNmU4YzgzMWIwOTg0NmE1YzQ4Y2NhMDhiODQ4zgACAADOAAIAANlPbWVzaDpjZG46Y2h1bms6Mzc3YzdiMGNjMjI5ZjQ3MDU2NzBkNjNkNTQ0YzI3NWI2NjNkZjZlOGM4MzFiMDk4NDZhNWM0OGNjYTA4Yjg0OJTZQDM3N2M3YjBjYzIyOWY0NzA1NjcwZDYzZDU0NGMyNzViNjYzZGY2ZThjODMxYjA5ODQ2YTVjNDhjY2EwOGI4NDjOAAQAAM4AAgAA2U9tZXNoOmNkbjpjaHVuazozNzdjN2IwY2MyMjlmNDcwNTY3MGQ2M2Q1NDRjMjc1YjY2M2RmNmU4YzgzMWIwOTg0NmE1YzQ4Y2NhMDhiODQ4lNlAMzc3YzdiMGNjMjI5ZjQ3MDU2NzBkNjNkNTQ0YzI3NWI2NjNkZjZlOGM4MzFiMDk4NDZhNWM0OGNjYTA4Yjg0OM4ABgAAzgACAADZT21lc2g6Y2RuOmNodW5rOjM3N2M3YjBjYzIyOWY0NzA1NjcwZDYzZDU0NGMyNzViNjYzZGY2ZThjODMxYjA5ODQ2YTVjNDhjY2EwOGI4NDiU2UAzNzdjN2IwY2MyMjlmNDcwNTY3MGQ2M2Q1NDRjMjc1YjY2M2RmNmU4YzgzMWIwOTg0NmE1YzQ4Y2NhMDhiODQ4zgAIAADOAAIAANlPbWVzaDpjZG46Y2h1bms6Mzc3YzdiMGNjMjI5ZjQ3MDU2NzBkNjNkNTQ0YzI3NWI2NjNkZjZlOGM4MzFiMDk4NDZhNWM0OGNjYTA4Yjg0OJTZQGEwYmI2OWY0NDg2MWViM2RiNTYwMjZhMWFiYmY5MWZmYTYyMTMyNTA4ZDVhMDZlMGQwZGRjNWNjMjZiZTMzZmPOAAoAAM4AAd4x2U9tZXNoOmNkbjpjaHVuazphMGJiNjlmNDQ4NjFlYjNkYjU2MDI2YTFhYmJmOTFmZmE2MjEzMjUwOGQ1YTA2ZTBkMGRkYzVjYzI2YmUzM2ZjkqFhoWKDojEwoXilYWxwaGGheaEyoXo=" + "messagePackBase64": "mrVQaXhlbCBIZXJvZXMvSMO8dCDimIOlYXNzZXSg2UBjZjhhNDUzYTM2OTgxMzIxOTc3MzU1NDQ4ZDI0ZGY0OWJjZWZlZTIyYmU2ZTM3MzBiOTE0MGU4N2MyOTAxM2Y5zgAL3jG/YXBwbGljYXRpb24veC1jdWx0bmV0LXRzLXZlY3RvctkhMjAyNi0xMC0xOVQwMDowMDowMC4wMDAwMDAwKzAwOjAwlpXZQDM3N2M3YjBjYzIyOWY0NzA1NjcwZDYzZDU0NGMyNzViNjYzZGY2ZThjODMxYjA5ODQ2YTVjNDhjY2EwOGI4NDgAzgACAADZT21lc2g6Y2RuOmNodW5rOjM3N2M3YjBjYzIyOWY0NzA1NjcwZDYzZDU0NGMyNzViNjYzZGY2ZThjODMxYjA5ODQ2YTVjNDhjY2EwOGI4NDihMJXZQDM3N2M3YjBjYzIyOWY0NzA1NjcwZDYzZDU0NGMyNzViNjYzZGY2ZThjODMxYjA5ODQ2YTVjNDhjY2EwOGI4NDjOAAIAAM4AAgAA2U9tZXNoOmNkbjpjaHVuazozNzdjN2IwY2MyMjlmNDcwNTY3MGQ2M2Q1NDRjMjc1YjY2M2RmNmU4YzgzMWIwOTg0NmE1YzQ4Y2NhMDhiODQ4pjEzMTA3MpXZQDM3N2M3YjBjYzIyOWY0NzA1NjcwZDYzZDU0NGMyNzViNjYzZGY2ZThjODMxYjA5ODQ2YTVjNDhjY2EwOGI4NDjOAAQAAM4AAgAA2U9tZXNoOmNkbjpjaHVuazozNzdjN2IwY2MyMjlmNDcwNTY3MGQ2M2Q1NDRjMjc1YjY2M2RmNmU4YzgzMWIwOTg0NmE1YzQ4Y2NhMDhiODQ4pjI2MjE0NJXZQDM3N2M3YjBjYzIyOWY0NzA1NjcwZDYzZDU0NGMyNzViNjYzZGY2ZThjODMxYjA5ODQ2YTVjNDhjY2EwOGI4NDjOAAYAAM4AAgAA2U9tZXNoOmNkbjpjaHVuazozNzdjN2IwY2MyMjlmNDcwNTY3MGQ2M2Q1NDRjMjc1YjY2M2RmNmU4YzgzMWIwOTg0NmE1YzQ4Y2NhMDhiODQ4pjM5MzIxNpXZQDM3N2M3YjBjYzIyOWY0NzA1NjcwZDYzZDU0NGMyNzViNjYzZGY2ZThjODMxYjA5ODQ2YTVjNDhjY2EwOGI4NDjOAAgAAM4AAgAA2U9tZXNoOmNkbjpjaHVuazozNzdjN2IwY2MyMjlmNDcwNTY3MGQ2M2Q1NDRjMjc1YjY2M2RmNmU4YzgzMWIwOTg0NmE1YzQ4Y2NhMDhiODQ4pjUyNDI4OJXZQGEwYmI2OWY0NDg2MWViM2RiNTYwMjZhMWFiYmY5MWZmYTYyMTMyNTA4ZDVhMDZlMGQwZGRjNWNjMjZiZTMzZmPOAAoAAM4AAd4x2U9tZXNoOmNkbjpjaHVuazphMGJiNjlmNDQ4NjFlYjNkYjU2MDI2YTFhYmJmOTFmZmE2MjEzMjUwOGQ1YTA2ZTBkMGRkYzVjYzI2YmUzM2ZjpjY1NTM2MJKhYaFig6IxMKF4pWFscGhhoXmhMqF6" }, { "label": "ts empty body", @@ -72,7 +72,7 @@ } }, "recordKey": "mesh:cdn:artifact:pixel-heroes-hüt:unversioned:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "messagePackBase64": "mrVQaXhlbCBIZXJvZXMvSMO8dCDimIOlYXNzZXSg2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AL9hcHBsaWNhdGlvbi94LWN1bHRuZXQtdHMtdmVjdG9y2SEyMDI2LTEwLTE5VDAwOjAwOjAwLjAwMDAwMDArMDA6MDCRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAA2U9tZXNoOmNkbjpjaHVuazplM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1kqFhoWKDojEwoXilYWxwaGGheaEyoXo=" + "messagePackBase64": "mrVQaXhlbCBIZXJvZXMvSMO8dCDimIOlYXNzZXSg2UBlM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1AL9hcHBsaWNhdGlvbi94LWN1bHRuZXQtdHMtdmVjdG9y2SEyMDI2LTEwLTE5VDAwOjAwOjAwLjAwMDAwMDArMDA6MDCRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAA2U9tZXNoOmNkbjpjaHVuazplM2IwYzQ0Mjk4ZmMxYzE0OWFmYmY0Yzg5OTZmYjkyNDI3YWU0MWU0NjQ5YjkzNGNhNDk1OTkxYjc4NTJiODU1oTCSoWGhYoOiMTCheKVhbHBoYaF5oTKheg==" }, { "label": "ts single chunk, versioned", @@ -93,7 +93,7 @@ } }, "recordKey": "mesh:cdn:artifact:pixel-heroes-hüt:v1-beta:89f4ff56a25dd1db06a4ce6033603775d705fb96f30f8693733fef602a1ca532", - "messagePackBase64": "mrVQaXhlbCBIZXJvZXMvSMO8dCDimIOlYXNzZXSnVjEgQmV0YdlAODlmNGZmNTZhMjVkZDFkYjA2YTRjZTYwMzM2MDM3NzVkNzA1ZmI5NmYzMGY4NjkzNzMzZmVmNjAyYTFjYTUzMs0D6L9hcHBsaWNhdGlvbi94LWN1bHRuZXQtdHMtdmVjdG9y2SEyMDI2LTEwLTE5VDAwOjAwOjAwLjAwMDAwMDArMDA6MDCRlNlAODlmNGZmNTZhMjVkZDFkYjA2YTRjZTYwMzM2MDM3NzVkNzA1ZmI5NmYzMGY4NjkzNzMzZmVmNjAyYTFjYTUzMgDNA+jZT21lc2g6Y2RuOmNodW5rOjg5ZjRmZjU2YTI1ZGQxZGIwNmE0Y2U2MDMzNjAzNzc1ZDcwNWZiOTZmMzBmODY5MzczM2ZlZjYwMmExY2E1MzKQgA==" + "messagePackBase64": "mrVQaXhlbCBIZXJvZXMvSMO8dCDimIOlYXNzZXSnVjEgQmV0YdlAODlmNGZmNTZhMjVkZDFkYjA2YTRjZTYwMzM2MDM3NzVkNzA1ZmI5NmYzMGY4NjkzNzMzZmVmNjAyYTFjYTUzMs0D6L9hcHBsaWNhdGlvbi94LWN1bHRuZXQtdHMtdmVjdG9y2SEyMDI2LTEwLTE5VDAwOjAwOjAwLjAwMDAwMDArMDA6MDCRldlAODlmNGZmNTZhMjVkZDFkYjA2YTRjZTYwMzM2MDM3NzVkNzA1ZmI5NmYzMGY4NjkzNzMzZmVmNjAyYTFjYTUzMgDNA+jZT21lc2g6Y2RuOmNodW5rOjg5ZjRmZjU2YTI1ZGQxZGIwNmE0Y2U2MDMzNjAzNzc1ZDcwNWZiOTZmMzBmODY5MzczM2ZlZjYwMmExY2E1MzKhMJCA" }, { "label": "ts whitespace version, padded kind and mime", @@ -114,7 +114,7 @@ } }, "recordKey": "mesh:cdn:artifact:pixel-heroes-hüt:unversioned:89f4ff56a25dd1db06a4ce6033603775d705fb96f30f8693733fef602a1ca532", - "messagePackBase64": "mrVQaXhlbCBIZXJvZXMvSMO8dCDimIOnIGJ1aWxkIKIgINlAODlmNGZmNTZhMjVkZDFkYjA2YTRjZTYwMzM2MDM3NzVkNzA1ZmI5NmYzMGY4NjkzNzMzZmVmNjAyYTFjYTUzMs0D6L0gIHRleHQvcGxhaW47IGNoYXJzZXQ9dXRmLTggINkhMjAyNi0xMC0xOVQwMDowMDowMC4wMDAwMDAwKzAwOjAwkZTZQDg5ZjRmZjU2YTI1ZGQxZGIwNmE0Y2U2MDMzNjAzNzc1ZDcwNWZiOTZmMzBmODY5MzczM2ZlZjYwMmExY2E1MzIAzQPo2U9tZXNoOmNkbjpjaHVuazo4OWY0ZmY1NmEyNWRkMWRiMDZhNGNlNjAzMzYwMzc3NWQ3MDVmYjk2ZjMwZjg2OTM3MzNmZWY2MDJhMWNhNTMykIA=" + "messagePackBase64": "mrVQaXhlbCBIZXJvZXMvSMO8dCDimIOnIGJ1aWxkIKIgINlAODlmNGZmNTZhMjVkZDFkYjA2YTRjZTYwMzM2MDM3NzVkNzA1ZmI5NmYzMGY4NjkzNzMzZmVmNjAyYTFjYTUzMs0D6L0gIHRleHQvcGxhaW47IGNoYXJzZXQ9dXRmLTggINkhMjAyNi0xMC0xOVQwMDowMDowMC4wMDAwMDAwKzAwOjAwkZXZQDg5ZjRmZjU2YTI1ZGQxZGIwNmE0Y2U2MDMzNjAzNzc1ZDcwNWZiOTZmMzBmODY5MzczM2ZlZjYwMmExY2E1MzIAzQPo2U9tZXNoOmNkbjpjaHVuazo4OWY0ZmY1NmEyNWRkMWRiMDZhNGNlNjAzMzYwMzc3NWQ3MDVmYjk2ZjMwZjg2OTM3MzNmZWY2MDJhMWNhNTMyoTCQgA==" } ] } diff --git a/contracts/cultnet/cultnet.witness-artifact-bundle.schema.json b/contracts/cultnet/cultnet.witness-artifact-bundle.schema.json index 6dd6971d..936a0a04 100644 --- a/contracts/cultnet/cultnet.witness-artifact-bundle.schema.json +++ b/contracts/cultnet/cultnet.witness-artifact-bundle.schema.json @@ -59,6 +59,7 @@ ], "additionalProperties": false, "properties": { + "id": { "type": "string", "minLength": 1 }, "role": { "type": "string", "minLength": 1 }, "schemaId": { "type": "string", "minLength": 1 }, "schemaVersion": { "type": "string", "minLength": 1 }, @@ -74,6 +75,7 @@ ], "additionalProperties": false, "properties": { + "id": { "type": "string", "minLength": 1 }, "role": { "type": "string", "minLength": 1 }, "uri": { "type": "string", "minLength": 1 }, "mediaType": { "type": "string", "minLength": 1 }, @@ -92,6 +94,7 @@ ], "additionalProperties": false, "properties": { + "id": { "type": "string", "minLength": 1 }, "stage": { "type": "string", "minLength": 1 }, "startedAt": { "type": "string", "minLength": 1 }, "completedAt": { "type": "string", "minLength": 1 }, diff --git a/docs/c2a-land-map.md b/docs/c2a-land-map.md new file mode 100644 index 00000000..3f31f789 --- /dev/null +++ b/docs/c2a-land-map.md @@ -0,0 +1,441 @@ +# C2a land map + +What C2a (`hands/variants-c2a`, a5c7954e) must still prove to merge to main, measured +on 2026-10-22 against main 7a8933b0 and the 2.0.0 release candidate +`hands/cultlib-unity-release-2.0.0` (5b5d8573, which carries f91e30ed). The land cut +is the merge plus the gate. Everything else is classified below. Governing ruling: +`cultlib-gaps:ruling:release-now-c2a-bumps-format`. + +## Cut sequence to main (2026-10-27, branch head 7c7cc799) + +Landed since the facts below: `land-c2a` (h1), `land-c2a-merge-api-docs` (h1-h6), the 2.0.1 and +cultcache-rs 0.3.1 merges (5baa02fa, f54472e5), `land-c2a-gate` r4 (h1), `store-gate-one-owner` +(h1-h7, Soul s6) and `restamp-on-write` (h1-h2, 8ee66714..7c7cc799, Soul s2 holds). One static `Owns` +decides ownership for cache and cacheless writers, and the High finding +`cut-land-c2a-gate.s1.reflist-record-read-only-at-head` is Fixed by 7c7cc799. origin/main is 73e684b1: +21 commits past the merge base 02cf04b5, all docs and research (15 files, +1879, none under src/, +tests/ or packages/, none on a path the branch changed). What stands between the branch and main, in +order: + +1. **claimant-one-owner** (spec r1 at 7c7cc799, Hands then Soul). Restamp's Soul s2 left four Lows + on the landing path. Which entry reads a persisted entry is chosen twice (`Claimant` at + CultCache.cs:703-750, `ClaimantEntry` at :797-799), and so a cacheless writer handed registry {A,B} + overwrites a record that a cache with {A,B} holds read-only (probe X4). The cut keeps one `Claimant` + precedence over a by-id/by-name lookup that the registry or a single entry supplies. A cacheless + writer with a registry then takes its claimant from that registry. The cut also pins the + unpublished-id case and fixes the stale test comment. It closes `claimant-selection-duplicated`, + `cacheless-inverts-registry-claimant`, `unpublished-id-guard-unpinned` and `stale-flag-copy-comment`. + It is kept apart from the gate so that Soul can falsify it alone. Its files (CultCache.cs, the + persistence contract, RestampOnWriteTests.cs) are disjoint from the gate's, so it can run before or + beside step 2. The second cut to land rebases. +2. **land-c2a-gate** (spec r7 at 7c7cc799, Hands h2 then Soul). The ref-list gate does the same write + at 2.0.1 and at head and compares schema ids. The committed vectors are pinned byte for byte under a + fixed clock, and every stage counts its cases. r7 drops r6's head-only restamp assertions, which + `RestampOnWriteTests.TheCommitted1060VectorsListRecordIsWrittenUnderTheHoldersOwnId` already pins. + It also drops the red-at-193cf1c3 run, because restamp is measured. The cut closes + `single-vector-hand-edit-passes`, `gate-green-with-no-refusal-probes` and follow-up + `gate-reads-1x-written-store`. +3. **Ledger** (Self, no Hands; can run now, beside steps 1 and 2). Each of these is in force though + its fix has landed: + - The seven `cut-land-c2a.s1.*` findings. `land-c2a-merge-api-docs` carried their pins (source reads + 2026-10-27: GameCult.Caching grants InternalsVisibleTo only to `.Tests` and `.MessagePack`, the + residual being follow-up `caching-messagepack-ivt`; PreCut2StoreFormatTests pins "with a suffix + of"). Close each one against its commit, or defer it. `stale-unity-plugin-dlls` defers to the + Unity 3.0.0 release cut (the plugins are 2.0.1 builds, 1eeb1a01). + - `cut-c2a-write-set.s1.untouched-dropped-slot-resurrects` and follow-up `slot-reuse-contract-doc`. + Ruling `slot-reuse-documented` decided the behaviour, and the paragraph is in + cultcache-schema-compatibility.md:64-66. + - `cut-restamp-on-write.s1.websockets-tests-unreported`, Fixed by h2 (s2 promise + `websockets-tests-reported` holds). + - Follow-up `gate-bound-test-tighten`, carried by restamp r2 (50362447, a768b821). Close it if + restamp's Soul passes measured it; otherwise route it. + - After steps 1 and 2 hold: the four s2 Lows, the two remaining `cut-land-c2a-gate.s1` Lows and + `gate-reads-1x-written-store`. +4. **Consumer pins** (ruling `unpinned-consumers-at-land-3`, follow-up `consumer-pins-before-land`). + Brokkr, Aquarium, Ymir, AquaSynth and Mimir have no `CultLibRevision`/`VerifyCultLibRevision` guard, + and each builds against the sibling `..\CultLib` checkout. In its own repo, each adopts Aetheria's + guard at the 2.0.0 tag, with one Hands per repo, before the merge reaches main. The pins touch no + CultLib file, so they can start now and run in parallel with steps 1-3. Eve and Gjallar get no pin + (ruled). The step is done when each repo's guard fails a build against a CultLib checkout at + another revision and passes at the tag, and the follow-up is resolved. +5. **Main into the branch, then full verification.** After steps 1, 2 and 4, `git merge origin/main` + into `hands/variants-c2a`. It should be clean, because main's commits since 02cf04b5 are docs and + research only. Re-check that before merging (`git diff --stat 02cf04b5 origin/main -- src tests + packages unity`). Then run every suite at the merged head on Yggdrasil: Caching, Mesh, Networking, + Geometry, WebSockets, ConsumerBuild, InteropPeer, cultcache-ts/-rs/-py, cultnet-ts and cultmesh-ts. + Run `old-reader-refusal.sh gate` for csharp, typescript, python and rust. If main gained code by then, + a Soul pass covers the merge. +6. **Land.** The land is `git merge --no-ff`, never a squash, so the SHAs that every report and verdict cites stay reachable from main (skip the branch's non-building commits with `git bisect --first-parent`). Steps 1 and 2 closed at 723b7996 (claimant-one-owner, land-c2a-gate); step 5 closed at the merge commit 3157d4b3 (Soul `cut-land-c2a-main.s1` holds). Step 3 closed except `cut-land-c2a.s1.stale-unity-plugin-dlls` and `cut-land-c2a-gate.s2.mutation-table-reads-gate-as-zero`, which wait on the 3.0.0 release cut and the Eureka tools respectively; step 4 closed with `consumer-pins-before-land` (pins since 2026-10-22). Run `git merge --no-ff hands/variants-c2a` onto main, with an empty diff against the + verified head. There is no tag: tags, the Unity plugin rebuild and `declared-range-gate-own-version` + belong to the 3.0.0 release cut. Open follow-ups `c2a-load-slower` (load 2 to 2.3 times slower than + main on a synthetic Aetheria catalog) and `drift-write-ownership-leaves` do not say "before land". + `c2a-load-slower` still needs a reason or a fix before the 3.0.0 release, under + `cultlib-gaps:ruling:database-shaped-built-for-load`. + +## Facts + +**1. The merge.** A trial `git merge origin/main` on a5c7954e gives 12 conflicted files and +33 hunks of about 950 marker lines (CultCache.cs 17 hunks/196; cult-cache.test.ts 337; +single-file-messagepack-backing-store.ts 4/106; rs lib.rs 2/51; VariantTests 51; three +CHANGELOG add/adds; store-composition contract 32; Directory store 14; InteropPeer 21). +Main moved 4.8k lines in the cache runtimes since base 1a12bc2d. Three semantic collisions +are not textual: + +- *Durable record identity has two owners.* Main added `CultStoredDocument.PersistedSchemaId` + and `CacheBackingStore.MarkWritten` (73d438db, afd799ba, 3f6dda8b). Conditional commit + compares the persisted schema id plus storedAt. C2a already owns this as + `StoredSchemaId` + `StoredVersion` (SHA-256 of the stored bytes, which names the schema + id and storedAt) set by `CultCache.Wrote`. After the merge, 9 references in + ConditionalCommitTests and the contract name main's owner. The version is the owner. + PersistedSchemaId and MarkWritten are deleted, and main's +309 test lines are ported as + pins against StoredVersion. 2.0.0 ships MarkWritten as protected API, and C2a also + removes `SerializeSnapshot`/`DeserializeSnapshot` overrides and changes `ConditionsHold`. + C2a is therefore a major release (3.0.0), which semver forces. +- *TS store commit moved.* Main's `store-file.ts` (0e6452c9, 0010ee92; 391 lines) now owns + commit, replace and the `.lock`. C2a's v3 read and keep-v3-on-rewrite + (`STORE_FORMAT_ELEMENT_IDS`) lives in the old store file and must move into store-file.ts. +- *Rust decode door.* Main's cultcache-rs 0.3.0 grown decoder (`decode_stack`, fc630d3b) + must sit under C2a's `readable_store_format` and v3 `decode_store_file`. + +**2. The gate already holds; it needs a test, not a format change.** C2a stamps +`cultcache.store.v3` (single file) or `...v5.directory-content-addressed-pages` exactly when +a record holds an id (`HeaderFor`, CultCache.cs:3543). It also marks when a copied record +decodes holding one (`CopiedHoldIds`, :4285). The 2.0 C# reader accepts only v1/v2 +(CultDocumentMessagePackSerialization.cs:229) and directory v4 (Directory store :452). +Main's TS, Rust and Python readers accept only v1 (store-format.ts `isV1Snapshot`, lib.rs:1274, +stores.py:192). Probe on Yggdrasil at 5b5d8573 plus C2a's `v3-base.msgpack` (job +`gate1.log`): 4/4 pass. The single file is refused naming v3, and a later flush leaves it +byte-identical. A v5-stamped directory store is refused naming v5, and its 4 files are +byte-identical after a later add and flush. **The pin:** C2a already ships +`tests/vectors/document-variants-c2a/old-reader-refusal.sh`, which stages an old revision +plus probes on a scratch ref and runs them per runtime. The land cut sets its +`OLD_READER_REV` default to the tag `cultlib-unity-v2.0.0`. It also adds the never-rewritten +assertion and the v5 directory case to the C# probe, and runs all four runtimes at the +tag. If the tag is absent, the land waits for it. The tag, not a package, is the pin: the tag's +tree holds every runtime's released reader. + +*Outside C2a's reach (release session):* in 2.0, `CultMesh.WriteSingleFileDocument`/ +`WriteSingleFileDocumentPayload` (CultMeshSingleFileDocuments.cs:211-245 at 7a8933b0) build +a fresh one-record v1 snapshot and replace the file without reading it. A 2.0 Mesh write +to a C2a v3 path therefore discards it whole, ids and other records included, and no header +gate can stop a writer that never reads. C2a's version reads under the lock and refuses. +Fix it in 2.0.0 before tagging, or record it. + +**5. Consumers.** Aetheria pins a CultLib revision (`VerifyCultLibRevision`) and Unity tags. +Ghostlight, Eve's Rust crates, Huginn and the Brokkr daemon pin cultcache-rs by git rev. +None of them moves when main moves. Two C# consumers build against an unpinned sibling +`..\CultLib` checkout: Eve `GameCult.Eve.Surface.csproj:16` and Brokkr's three csprojs. +Eve.Surface has `[MessagePackObject]` element arrays with no `[CultElementId]` +(`EveAssetCatalogEntry[]`, `EveEntitySoaBuffer[]`, `EveInputActionDocument[]`, +`IReadOnlyList`, 0 ids in the repo), so C2a registration refuses its +documents as soon as the checkout pulls the land. Grep finds no object lists in Brokkr's +`[CultDocument]` files (not build-verified). Removed C2a API (`Options`, `AsSchemaAlias`) is +unused by Eve and Brokkr. Fork: `variants:question:unpinned-consumers-at-land`. + +## Classification + +| Item | Class | Why | +|---|---|---| +| c2a-format-version-gate | **a** | The gate test above. No new header: v3/v5 already refuse. | +| slot-reuse-contract-doc | **a** | The ruled consequence of the write set C2a lands. One paragraph in cultcache-schema-compatibility.md, absent at a5c7954e (grep). | +| foreign-ids-under-v1-header | **b** | Obsoleted by fb065f0d (write-time `CopiedHoldIds`, after the finding) for every C# writer with the type. The residual (non-C# or opaque writers) is what `non-csharp-writers-mark-ids-2` ruled to carry, and the contract states it (persistence-format.md:434-439). The gate changes neither side, so no fork remains. Resolve as Superseded by that ruling. | +| custom-resolver-element-ids | **b** | Fixed by b091d21f (19:52Z, after the 18:27Z follow-up): `ListElementsClaimedByAResolverAreRefused`, `AResolverClaimingTheListTypeOfAnObjectListIsRefused`. s6 holds on the claim loop. | +| codec-door-proven-mismatch | **b** | Done in h6 57fdf70c (s6 `only-proven-mismatch` Holds; Geometry 76/0). | +| consumer-element-id-sweep-r2 | **c** | Each repo's sweep is its own work (q3). Only the unpinned trackers matter at land, and that is the question below. Aetheria bumps a pin on purpose (C4). | +| schema-discovery-holder-recogniser | **c** | `TryGetDictionaryRefTypes` is identical on main (CultCache.cs:1288). It describes record-ref dictionaries and is not on the id path. Changing it moves schema hashes, which is its own epoch event. | +| copy-check-memo | **c** | Performance only (about 3x flush on unmarked stores of 20k records). Correct as is. | +| string-key-field-and-shadow-pins | **c** | No wrong behaviour at 64ec14f9. These are test pins and may ride the land cut if Hands touches the file. | +| declared-range-gate-own-version, range-check-in-ci | **c** | The script exists only on C2a and runs in no CI. It guards a TS publish, not a merge. Fix both before the first TS release from main after the land. | +| s6 Unproven: Unity/IL2CPP identity, other MessagePack versions | **c** | The merge reaches no Unity build. Unity sees C2a only at the 3.0.0 tag, so measure it in the Unity release cut (operator Unity check, C3). | +| multi-store-commit-atomicity, store-replace-cut, reader-refuses-duplicate-keys, soft-flush-all-runtimes, writer-dangling-link-tolerance, unique-index-cut, py-shared-write-lock, rust-lock-open-waits, ts-lock-test-hardening, c0/legacy/directory vectors, culture-test, wire-id-cuts, remaining-variant-cuts, golden-path, paged store | **c** | None says "before land". Each is pre-existing on main or C2a-independent. None lets a released reader strip ids or corrupt a C2a store. | + +## The land cut's shape (transcribe once the question is ruled) + +- **Base.** Merge main at or after the `cultlib-unity-v2.0.0` tag into `hands/variants-c2a`. + Versions stay main's. CHANGELOGs get Unreleased entries, and the Unity entry says major. +- **Deletes and collapses.** Delete `PersistedSchemaId`, `PersistedIdentity`, + `CultStoredDocument.MarkWritten` and `CacheBackingStore.MarkWritten` (main, about 60 lines), + and the store-composition contract's MarkWritten paragraph. C2a's `StoredVersion`/`Wrote` + is the one owner. In TS, delete C2a's header handling in + single-file-messagepack-backing-store.ts and move it into store-file.ts. In Rust, C2a's v3 + decode goes through the grown decoder. +- **Adds.** Retarget old-reader-refusal.sh to the tag. Add the C# probe's never-rewritten + assertion and the v5 directory case (as in gate1.log). Add the slot-reuse paragraph. + Port main's ConditionalCommitTests to the version identity. +- **Verification.** Full suites: Caching, Mesh, Networking, Geometry, WebSockets, + ConsumerBuild, InteropPeer, cultcache-ts/-rs/-py, cultnet-ts and cultmesh-ts, all on + Yggdrasil. Run old-reader-refusal.sh for all four runtimes at the tag, and it must refuse. + Negative greps over the merged tree: `PersistedSchemaId|MarkWritten`, conflict markers. + Mutation: drop the v3/v5 stamp in `HeaderFor` and the gate probe must fail. +- **Estimate.** About 950 conflict lines resolved. Net -60 to -150 from the identity + collapse and TS de-duplication, +80 to +120 tests and docs. One Hands pass of 100-150 + calls, then one Soul pass. +- **Precondition (question).** Eve.Surface, and Brokkr if confirmed, are pinned or swept + before the merge reaches main. + +## Store gate: attachment is the attach's hold (store-gate-one-owner r2) + +Spec `variants:cut_spec:cut-store-gate-one-owner.r2` (base ec94ab98), question +`variants:question:pulled-variant-codec`. It closes Soul s3's +`attach-window-cacheless-write` (Medium) and `codec-loan-pinned-by-reflection` (Low). + +**Facts (source reads at ec94ab98 and a probe, 2026-10-25).** + +- The window. AddBackingStore holds the cache's gate, then the store's gate through + `InStoreHold`, lends the registry and codec, sets `Loaded`, adds the store to `_stores`, + pulls, and only then sets `store.Cache` (CultCache.cs 2268-2312). `InStoreHold` lets any + call on the same store's gate re-enter it (2825-2831). So a journal the pull's Admit runs + on the attaching thread can call `store.WriteDocument`: `Gate(cacheless)` guesses null, + re-enters the store's gate, finds `Cache == null` and runs the write, and its `finally` + calls `UseRegistry(null, null)` (Soul s3 P1, full1.log lines 49-53). Other threads are not + in the window: they queue on the store's gate and see either the published cache or a + store that the failed attach put back. +- The codec loan has no reader on the attach path. A pulled variant's codec is read only + by `CultStoredDocument.HoldsIds` (543-557). Header decisions read `HoldsIds` only on + staged entries (single-file 4318, 4447 and 4472; directory store 362), and a staged entry + is always the document the cache pushed (`Push` sets `Entries[key] = entry`, 4211-4215). + A single-file `PullAll` returns without loading while the store is dirty (4157-4159), so + a pulled copy is never staged. A refused write unstages its keys before it adopts the + reload. The directory store refuses variants (ruling `q6-directory-and-cultnet-refuse`; + DirectoryMessagePackBackingStore.cs 139-144 and 642-643). Admit re-resolves every pulled + variant with the cache's codec (CultCache.cs 3259). Soul's N4 mutant (the attach lends + no codec) was caught only by the reflection test, and h4's M2 only by a bare-store test + that sets `Loaded` and calls `UseRegistry` by hand. +- Probe (scratch clone `imagination-store-gate-r2/clone`, commits 4663934e and c099b9e8 on + ec94ab98; Yggdrasil t1.log, t2.log and t3.log; 2026-10-25 around 20:50 CEST). The + probe did three things. `InStoreHold` refuses a cacheless call that re-enters the same + store's gate. The `_codec` field and the two-argument `UseRegistry` are deleted. The pull + builds variants with a null codec. Results: + - Mesh is 342/0/4. + - In the Caching suite, the only failures were two probe tests, both from authoring + errors: a key `AddAsync` minted, and an element id that was not 12 hex characters. + No existing test failed, `UserCodeInACachelessCallMayCallItsOwnStore` included. + - Soul's P1 as a test: the journal's write is refused with the new message. The attach + keeps its registry and the file holds only `a`. After the attach, a cache write, a + flush and `store.PullAll()` all succeed. + - A cacheless write nested in another on the same store is refused, the outer write + fails, and no file is written. + - Through a real attach, a store holding a name-override variant flushes an unrelated + commit with a `cultcache.store.v2` header. With no codec loan, the pulled copy does not + mark the store. + - Mutant `return cacheless` changed to `return false`, which is ec94ab98's behaviour, + fails both new gate tests. In that mutant the nested write succeeds and the file exists. + +**Rationale.** + +- Attachment has one fact again. While the attach runs, the store's gate is the attach's + hold, and `Cache` is published when that hold ends. A cacheless write is a write the + store owns outright, so it must be the outermost hold of its store's gate. One more + argument to the existing owner of per-thread gate facts (`InStoreHold`) refuses it. No + field mirrors `Cache`, and `Loaded` and `_registry` are not read. Precedent: + `ConcurrentHashMap.computeIfAbsent` refuses a mapping function that updates the same + map with `IllegalStateException("Recursive update")` since Java 9. That is from + memory, not fetched this pass; the 2026-10-23 prior-art note covers the documented + restriction in its counter-positions. Non-cacheless store calls from user code still + re-enter: `UserCodeInACachelessCallMayCallItsOwnStore` stays green. +- The late publish stays. It is what bounds `Gate` at two passes (s2's finding). Publishing + early would bring back the failure-rate-bounded retry. +- The codec loan goes: it is a value nothing on the attach path reads. A pulled variant + carries no codec and counts as holding ids (the documented conservative answer), and + the cache's resolved copy is what any header decision sees. This reverses the codec + half of Defaulted ruling `late-publish-needs-attach-codec`, so it goes to Self as + `variants:question:pulled-variant-codec`. The spec is written for the recommended option. +- Scar check: `git log -S` over CultCache.cs for `reentrant`, `outermost` and `_attaching` + finds nothing. `Cache?.Codec` appears in 1468a83a (h4) and 619e089f only. No earlier + re-entry refusal was built and deleted. +- Behaviour change, stated in the CHANGELOG: a cacheless write made from user code that + another cacheless write of the same store runs used to succeed, and its `finally` took + the outer call's registry. It is now refused. + +## Store gate: Cache is the one attachment fact (store-gate-one-owner r3) + +Spec `variants:cut_spec:cut-store-gate-one-owner.r3` (base 0fa836cb), superseding r2. It +closes Soul s4's `attach-window-held-store-writes` (Medium), `cacheless-refusal-order-unpinned` +(Low) and `interop-consumer-suite-unrun` (Low). This is the third time the attach window has +beaten the rule (s3 from the cacheless side, s4 from the held side), so r3 moves the +decision instead of patching the next entry point. + +**Facts: every entry point that reads or decides attachment, at 0fa836cb.** Source reads of +CultCache.cs, DirectoryMessagePackBackingStore.cs and CultMeshSingleFileDocuments.cs. The +window column is a journal on the attaching thread, through a real attach of a store holding +`a`. Probe: scratch clone `imagination-store-gate-r3/clone`, commit b2330b6d (test R3Probes), +Yggdrasil `probe-base.log`, 2026-10-25 around 23:00 CEST. "Disagree" means that after the +attach, a commit of `later` and a flush, the attaching cache holds `a,later` and a fresh cache +over the same file reads something else. + +| Entry point (line) | Fact it reads | In the attach window at 0fa836cb | +| --- | --- | --- | +| Single-file `PullAll` (4156) | `Gate`: `Cache` guessed unlocked, then re-read under the re-entered store gate | Runs as detached; its `Loaded` re-enters the cache and is refused as a journal write. Agrees. | +| Single-file `Push` (4212) | same | Runs as detached and stages `b`. Disagree: fresh reads `a,b,later`. | +| Single-file `Delete` (4223) | same | Runs as detached and stages a removal the cache never made (dirty after the attach). | +| Single-file `PushAll` (4239), and `Dispose` (3722) through it | same, then in-body `Cache == null` (4531, 4543, 4588, 4603, 4611, 4661) | Runs as a cacheless writer. Push then PushAll writes `b` to the file inside the attach (disagree). Delete then PushAll is refused with the cacheless writer's text. On a variant store it is refused as "a writer with no CultCache" (Soul s4). | +| Single-file `CommitBatch` (4256) | same | Runs as detached and commits `b`. Disagree. | +| Single-file `JudgeChecks` (4568), `RefuseRemovalOfUndeclared` and `StageChecks` (4558, 3915) | same | Run as detached. Agree in the probe, but under the wrong gate's rules. | +| Single-file `ReadStoreSnapshot` (4687) | same | Runs as detached and reads the file. | +| `WriteRecord` (4342), `WriteDocument` (4366) | `InStoreHold`'s per-thread hold (cacheless re-entry), before `Cache` | Refused with the re-entry text (h5). Agree. | +| Directory `PullAll` (48) | `Gate`, as above | Refused as a journal write. Agree. | +| Directory `Push` (147), `Delete` (160), `CommitBatch` (172) | same | Run as detached. Disagree: fresh reads `a,b,later` after Push or CommitBatch, and `later` after Delete. | +| Directory `PushAll` (235) | same | Runs as detached. Push then PushAll and Delete then PushAll disagree. | +| Directory `JudgeChecks` (269), `RefuseRemovalOfUndeclared` (259) | same | Run as detached. Agree. | +| In-body reads: `StoredAtOver` clock (4028), refusal text and `DurableAt` (4531, 4543), declares and cacheless refusal (4588, 4603, 4611), `Wrote` (4661) | `Cache`, under the gate `Gate` chose | On the attaching thread they read null while the thread holds the cache's gate, so a held call runs as a cacheless writer. | +| `AddBackingStore` (2259-2314) | writes `Cache` under both gates, after the pull | The window's author: the store is attached for the pull (registry lent, `Loaded` set, in `_stores`) while `Cache` is null. | +| Mesh `CultMeshSingleFileDocuments` writes and reads (99, 143, 174, 226) | none of the attached store's | `StoreAt` (239) builds a new store per call, so they never touch an attached store object. A file it shares with an attached store is the file lock's and the write set's business. | +| `IsDirty`, `HasStagedChecks`, `ForeignRecords`, `LastSchemaMigrationReports`, `HeaderFor`, `HeaderForWrite` | none | Ungated values or static decisions. They read no attachment fact. | + +So during an attach, cacheless entry points decide attachment from the hold, and every held +entry point, plus every in-body read, decides it from `Cache`, which is still null. These are +two facts. + +**Probe of the chosen shape** (commit 4695390c on b2330b6d; `proto-full.log`). `AddBackingStore` +publishes `Cache` before the pull and takes it back in the same hold if the pull fails. `Gate` +becomes straight-line code: guess, then the store's gate, where `Cache` is settled, then the +settled cache's gate. Results: + +- All 21 probe cases are refused. The cacheless writes get the re-entry text and every other + case gets "A journal entered the cache it journals". After the flush, the attaching cache + and a fresh cache both read `a,later`. +- Caching: the only failures are the two tests that pin the late publish: + `AnAttachLendsTheRegistryAndPublishesTheCacheAfterThePull` (asserts no cache during the + pull) and `ACallWhoseGateChangesTwiceUnderItThrowsInsteadOfLooping` (asserts the old message + text; the tampering reader still throws). The churn, queued-reader and failed-attach tests + pass unchanged. +- Mesh and Networking are green. +- The cultcache-ts suite, which builds and drives the C# InteropPeer, passes on this shape: + 175 tests, 171 pass, 0 fail, 4 skipped. The C#, Rust and Python interop tests ran, the C# + edit-in-place one included (`proto-ts2.log`, commit f5f02053). Recipe `clone/r3ts.sh`: + image node:24-bookworm, apt python3-msgpack, dotnet-install for 10.0, rustup minimal. + +**Shapes weighed (rationale).** + +- (a) Publish `Cache` before the pull, as a616431e did. Chosen. What s2 found was not that an + early publish is unbounded. It was that `Gate` re-read `Cache` *unlocked* after a stale + guess, so a churn of failing attaches could hand it a new stale guess every pass. The fix is + where `Gate` reads on its second step. Under the store's gate, `Cache` is settled: an attach + publishes and takes it back inside one hold of that gate, and no other thread can hold that + gate during the attach. A settled cache is permanent, because there is no detach. So `Gate` + needs at most three acquisitions, with no loop and no counter: the guess's gate, the store's + gate, and then the settled cache's gate, which cannot miss. This deletes a fact rather than + reconciling two. During the attach the store *is* attached, so every entry point, cacheless + or held, and every in-body read sees one answer. A journal's call is then refused by the rule + journals already have (AddJournal: "any hold on this cache while a journal runs throws"), + exactly as it is after the attach. Prior art: `ConcurrentHashMap.computeIfAbsent` installs a + ReservationNode in the bin *before* running the mapping function, so a recursive update on + the same thread finds the owner and throws, and on failure the reservation is removed (see + `docs/research/2026-10-23-store-lock-user-code-prior-art.md` item 4 on main: the re-check + after locking, and the ReservationNode on the empty-bin path). The validate-after-acquire in + `Gate` is the same item. +- (b) Keep the late publish and make the attach's hold the one fact for every entry point. + Rejected. `InStoreHold` knows that this thread holds the store's gate, not *whose* hold it is. + Refusing every same-store re-entry kills `UserCodeInACachelessCallMayCallItsOwnStore` (r2's + R5), so (b) needs an "attaching" marker beside `Cache`. That is the second fact that s3 and + s4 rule out, plus every in-body `Cache == null` read would need to consult it as well. +- (c) Run no user code on the attaching thread under the store's hold during a pull: defer the + pull's journals to after the hold (open calls). Rejected as insufficient on its own. Getters + and constructors still run during the pull's decode, so the window stays open to any user + code that is not a journal. It also moves journal timing, which is a documented contract + ("right after an admission is applied"). +- Scar check: `git log -S` over CultCache.cs finds `store.Cache = null` and `MaxStaleGuesses` + only in ba658965 and 1468a83a. The early publish was not removed for a correctness reason of + its own: 1468a83a removed it to bound `Gate` and to lend the codec, and the codec loan is gone + (`pulled-variant-codec`). No settled read under the store's gate was built before. The + precedent recipe (no root filter) finds no same decision. +- Not a product fork. A journal's store call during an attach now throws, as it already does + after the attach. Cacheless writes on the attaching thread stay refused, with the same text. + Getters and constructors during the pull see an attached store, as they do during any held + call. It changes the late-publish clause of Defaulted ruling `variants:ruling:pulled-variant-codec`, + so Self defaults a superseding ruling (recommended label `cache-published-for-the-pull`) + before dispatch. +- Residual, not in r3. `Push`, `Delete`, `PushAll` and `CommitBatch` are public, so user code + that is not a journal (another thread, or a getter during any held call) can stage records + behind an attached store's cache. Soul s4 shows this with `other-thread-push`: file `a,b`, + cache `a`. That is true before, during and after an attach, so it is not the window's + two-fact problem. Closing it changes public API (a store's writes taken only from its + cache), so it is a follow-up and an operator question for Self, not part of this cut. + +## Body facts: claimant-one-owner r2 (Imagination, 2026-10-27, base fc74cf6f) + +Why a revision: Soul beat the same rule three times (s1 claimant choice, s2 compatible ids, s3 shape). Each time a +cacheless write lent a registry let the registry's entry for the written schema id decide ownership beside the entry +the caller passed. Two descriptions of one type reached `Owns`; each fix (h2 schema-id compare, h3 subset of compatible +ids) compared one more facet of the two and left the rest open. r2 removes the second description from the lookup. + +### Where the two descriptions met (source read, fc74cf6f) + +- `OwnsOnDisk` cacheless arm (`CultCache.cs:4725-4740`): with a lent registry it called + `lent.Owns(id, catalog, declared, written)`, whose lookup is the registry's `_indexes` and whose `entryOf` is + `descriptor.ToCatalogEntry()` (`:824-825`). The claimant for the written schema id was the registry's descriptor, + so `ReadsWhole(persisted, claimantEntry)` (`:848`) judged shape against the registry's entry, and the claimant + precedence (`:756-806`) walked the registry's compatible ids. `written` (`:4734-4736`) then compared the two entries + by schema id plus a subset of compatible ids. +- `WriteDocument` (`:4495-4524`) takes its entry from the registry (`resolvedRegistry.GetRequired().ToCatalogEntry()`), + so its two descriptions are always identical. Only `WriteRecord(record, catalogEntry, holdsIds, registry)` (`:4471`) can + pass an entry that disagrees with the registry's entry for the same id. +- `CopiedHoldIds` (`:4600-4630`) also reads `AttachedRegistry`, to decide the v3 header from copied records. It is a + positive-only header decision, not ownership, and out of this cut. + +### Consumer audit (2026-10-27) + +- `WriteRecord` and `WriteDocument` are new in the Unreleased 3.0.0 entry; `main` has neither (`git show main:src/GameCult.Caching/CultCache.cs`). +- Per-repo `rg` over every git repo directly under `F:\Projects` for `.WriteRecord(`, `.WriteDocument(`, `WriteSingleFileDocument` + (Library, obj, bin, PackageCache excluded): hits only in CultLib and in vendored CultLib copies under Heimdall and StreamPixels + (`vendor\CultLib`, older pins without these methods). No consumer outside CultLib calls either method. +- Inside CultLib at fc74cf6f: Mesh `WriteSingleFileDocument` calls `WriteDocument(key, document, storedAt, registry)`; + Mesh's raw-payload path calls `WriteRecord(..., registry: null)` (`CultMeshSingleFileDocuments.cs:129, 226`). The only + `WriteRecord` calls with a registry are tests: `CachelessSingleFileTests.cs:453, 660` (a registry `{IdDeck}` that does + not hold the written entry `One` at all, used for the copied-record header), `RestampOnWriteTests.cs:595, 599`, and + Soul's probes. Packages: no TS, Python or Rust counterpart takes a registry (cultnet-rs `write_record` is an example helper). +- Consequence: the registry's documented role in `WriteRecord` is to describe the other records in the store. A shape + that derives the written entry from the registry (or refuses an entry the registry does not hold) breaks + `CachelessSingleFileTests.cs:660` and removes that role. + +### Candidate shapes weighed + +| Shape | One description? | Public API | Kills s2/s3 | Cost | +|---|---|---|---|---| +| A. Lent registry's entry decides; refuse a passed entry that disagrees | yes, after a door check | signature kept; new refusal | yes | an equality check over a redundant parameter; breaks the registry-describes-others use (`:660`) when the registry lacks the written type unless it falls back to the passed entry, which is two descriptions again | +| B. Passed entry is the only description of its schema id; the registry supplies other types only | yes, by construction | signature kept; doc only | yes (probed) | one lookup class gains the registry; one registry overload deleted | +| C. Overloads: `WriteRecord(record, catalogEntry, holdsIds)` and `WriteRecord(record, registry, holdsIds)` | yes, by type | changes a 3.0.0 signature | yes | loses the registry-describes-others use | +| D. Delete `WriteRecord`'s registry parameter | yes | changes a 3.0.0 signature | yes | loses the copied-record header decision for raw writes | + +Picked B. It is the only shape that keeps every documented use of the registry, needs no public signature change, +and deletes the comparison instead of extending it. It is ordinary scoping (an inner binding shadows the outer one +for its name), applied to schema ids. + +### Probes (Starfire, local filtered runs, 2026-10-27) + +Scratch clone of fc74cf6f at `/imagination-claimant-r2/clone`, Soul's `SoulClaimantProbe.cs`, +`SoulClaimant2Probe.cs`, `SoulClaimant3Probe.cs` copied into `tests/GameCult.Caching.Tests`. + +Base fc74cf6f (`probe-base.log`): +- P7 (registry {N lists own, A, B}; passed entry N with the list varied; record k under O, named like N): lent R / no + registry C for rows b, b2, e, g, h, k, l, l2; lent C for a, c, d, f, i, j. Confirms `subset-refuses-own-drift`. +- P9 (registry {NW: Name, Value, Colour}; passed NW minus Colour): lent C (restamped, Colour lost), no registry R. + Confirms `lent-shape-widens-passed-entry`. +- P6 (bare X1 entry, registry {X1} listing OX, record under OX): lent R, no registry R, cache C. h3 closed the id axis. +- P4: 12 states x 8 registries, violations=0. P1d: a cache {N,L} commits N over L's listed old id (`cache-replaces-other-types-record`). + +Prototype B (scratch commit 349ea3f3 on the clone, not for landing; `probe-protoB2.log`): `CatalogEntryLookup` takes +the written entries and an optional registry and drops every registry descriptor whose schema id a written entry has; +the registry's `Owns(schemaId, catalog, declared, mayClaim)` overload is deleted; the cacheless arm always asks +`CultDocumentRegistry.Owns(id, catalog, declared, new CatalogEntryLookup(writes, AttachedRegistry), entry => entry, +claimant => writes has claimant's schema id)`. +- P7: lent == no registry on all 14 rows (all C). +- P9: lent R, no registry R, wide entry C. +- P6: unchanged (R, R, C). P4: identical table to base, violations=0. P1a-f, P2a-d: identical to base. +- `RestampOnWriteTests`, `CachelessSingleFileTests` and the probes: 139/139 passed locally. +- Yggdrasil, 349ea3f3, `ygg-protoB-full.log`: GameCult.Caching.Tests 1180 passed / 0 failed / 2 skipped; + GameCult.Mesh.Tests 342 / 0 / 4. + +Negative greps, each tested at fc74cf6f (each matches there, and none may match after the cut, src and tests): +`claimant.CompatibleSchemaIds.All(` (1, the subset), `lent.Owns(` (1), `Func mayClaim) =>` +(1, the deleted overload), `CatalogEntryLookup(writes)` (3, one-argument construction). CHANGELOG and docs: +`never widens what the passed entry declares` (1), `with no compatible id the passed entry does not list` (1). + +### Not settled by r2 + +- `cache-replaces-other-types-record` (s2, PreExisting, cache arm): r2 touches only the cacheless lookup, so it does + not settle whether a cache may replace another registered type's record at a shared key. +- Whether a registry-less `WriteDocument` should consult `CultDocumentRegistry.Shared` for claimants at all (s1 + `shared-decides-registryless-write`, closed Fixed by documentation in h2). r2 keeps today's answer. +- `variants:finding:cut-restamp-on-write.s2.cacheless-inverts-registry-claimant` is still in force although r1 was + mapped to close it; Self should close or re-triage it after r2's Soul pass. diff --git a/docs/document-variants-cut.md b/docs/document-variants-cut.md index 854d6cbb..edf86d6a 100644 --- a/docs/document-variants-cut.md +++ b/docs/document-variants-cut.md @@ -91,6 +91,121 @@ Probes run 2026-10-12 from `F:\Projects\CultLib-c2afix` with git only (no builds `unstored-same-tick-unpinned`) caught at head. `git grep RawBytes de401056 -- src tests`: no match. Those findings were never closed in the mind. +### Body at 2026-10-15: element ids and the MessagePack walk (Modeling, from the Imagination and Soul passes) + +Probes ran on Yggdrasil (`mcr.microsoft.com/dotnet/sdk:10.0`, MessagePack 3.1.7 in every csproj) as probe +tests that were never committed: at `6afa8ed9` for I1 to I4, at `494ad9ab` for r4 and r5, at `6e70626b` for r6. +Logs and sources are under `F:\Projects\eureka-handoff-aux-2026-10-15\scratchpad-self-2026-10-14b-eureka\` +(`imag\`, `imag-r4\`, `imag-r5\`; Soul's `soul-c2a-id-holes-s5\` is in that session's scratchpad). The typed +record of each fact is in the mind: `variants:cut_spec:cut-c2a-element-id-holes.r5` and `.r6` (`first`), +verdicts `cut-c2a-element-id-holes.s5` and `.s6`, spec `cut-c2a-ids-header.r1` (`first`). This section keeps +what was run and what it showed. + +**What an id is on disk and in memory.** +- An id is an ordinary keyed string in the payload. A list element whose `[CultElementId]` member holds + `a1b2c3d4e5f6` and one whose plain `Tag` holds the same value encode to identical bytes + (`92A1649192AC61316232633364346535663607`; `c2a-ids-header` r1 probe, base `2774a7fe`). No reader can tell + from a record's bytes that it holds ids; only the store header says so. +- `StubArm`, a union arm that is not an object type, is never minted (`imag\imag1.log`, I1): for a document of + `StubArm` and `Unit8`, `Plan` counts 1, `StubArm.Id` stays `''`, `Unit8.Id` is minted, `Inspect` reports 0 + problems. +- Resolver over-mark (I2): a consumer `[assembly: CultCacheFormatterResolver]` claims a keyed arm with a + formatter that writes only `N`. The cache mints an id in memory, `Holds` is true and the header is stamped + `v3`, but the payload (`92A263319192009104`) carries no id, and a reopen mints a different one (`8f5e929888ca` + in memory, `a1ea2b683d98` after reload). The header claims ids the disk lacks. +- `[MessagePackFormatter]` on an element type is honoured (I3): the attribute formatter wrote + `92A26131919104`, no id, and `Inspect` reported 0 problems at that base. Registration refuses it from r5 on + (`declares [MessagePackFormatter]`). +- Python store (I4): `py-store/leaf-record.cc` (header `v1`, ids `000000000021/22`) loads in C# with + `HoldsIds = true`, `IdsInMemoryOnly = false`. A C# write that adds an unrelated record copies it forward, + the header stays `v1`, and the ids survive reload. cultcache-py, -ts and -rs have no declaration of which + member is an id, so they keep a `v3` they found and never decide one. +- Header narrowing (`c2a-ids-header` r1): per-record `StoredHoldsIds`, set from the type that decoded a + record, is falsified by the identical-bytes probe above. The durable header owns "these stored bytes hold + ids" at store granularity. A write that copies any record forward keeps a marked header; only a write that + encodes every record decides by content. `StoredHoldsIds` and `KnownIdFree` are deleted. +- Object list versus value list (r4, `imag-r4\probe1.log`, `probe2.log`): ids sit on object-list elements only + (ruling `q7-object-list-elements-only`). `Dictionary`, `ConcurrentDictionary`, `SortedDictionary`, + `IReadOnlyDictionary`, and `ILookup` of keyed arms round-trip with the id at + the element's own slot (e.g. `ConcDict` `81A16B92AC30313233343536373839616200`: key, then `[id, 0]`). A list + or map of scalars has no element to carry one and never enters the id path. + +**How MessagePack decides what it writes** (r5 `imag-r5\probe1.log`, `probe3.log`; Soul s5 `job2.log`). +- Member discovery, `MessagePack.Internal.ObjectSerializationInfo.CreateOrNull(Type, bool forceStringKey, + bool contractless, bool allowPrivate)`, is internal, and so is `StandardResolver.Resolvers`. The public + surface allows one test: `StandardResolver`'s formatter for `T` is reference-equal to the formatter one + public singleton hands out for `T`, each caching one instance per type. `DynamicObjectResolver.Instance` + matches keyed classes and structs, `DynamicUnionResolver.Instance` `[Union]` types, + `DynamicGenericResolver.Instance` List, arrays, maps, KeyValuePair, tuples, Lazy, Memory, ArraySegment and + Nullable, and `ImmutableCollectionResolver.Instance` (namespace `MessagePack.ImmutableCollection`) + `ImmutableArray` and `ImmutableList`. Asking `DynamicObjectResolver` for a collection type throws + `TypeInitializationException`, so a family probe swallows exceptions. +- Kind facts (`probe3`): a `[MessagePackObject]` class implementing only `IEnumerable` or + `IReadOnlyList` (`KBag`, `KNode`, `KRo`) is written by key (object resolver). One that is also + `ICollection`, a `List` subclass or a `Dictionary` subclass (`KC2`, `KColl`, `KList2`, `KDict`) is + written as a collection (generic resolver) and its `[Key]` members are dropped. +- Member facts (`probe3`; `ap` is `AllowPrivate`, `R` written, `W` read back). A member is written when it + has an integer `[Key]`, no `[IgnoreMember]`, and is a public field or a property with a public getter. It is + read back when it is a public non-readonly field, a property with a public setter (`init` counts), or a + `[SerializationConstructor]` parameter. At `ap=False`: `PG` (private getter) `R=False W=True`, `PS` + (private setter) `R=True W=False`, `GetOnly` `R=True W=False`, `InitOnly` both. `IdPriv` + (`Id { get; private set; }`) is written and not read back (wire `91AC62...`, back ``), and `IdCtor` is + `W=False`. At `ap=True` every one of these, `priv` included, is `R=True W=True`, except `GetOnly` + (`W=False`). +- AllowPrivate attribute form (Soul s5 `SoulS5Probe.C`, `ApTrueArm`): on a type declared + `[MessagePackObject(AllowPrivate = true)]`, `StandardResolver` writes private `[Key]` members and a + private-setter id round-trips. The oracle test calls `CreateOrNull(type, false, false, false)` and reports + none of them, so the walk's member set misses that form. +- Collection written as object (r6 `probe2.log`): a `[DataContract]` class implementing only `IEnumerable` + over an id-carrying arm (`DcSeq`) is a collection in the walk, yet `StandardResolver`'s formatter is + `DynamicObjectResolver`'s (wire `91 91 91 92 AC..`, written as an object). A `[DataContract]` `List` + subclass (`DcColl`) is written by the generic resolver, and the object resolver hands out a different + instance for it. For `DcBad` (two members on one `Order`) `StandardResolver` uses the generic resolver, and + Soul s5 measured both `DcColl` and `DcBad` passing the door. +- Geometry (r6 `probe2.log`, `probe3.log`): Geometry has a direct MessagePack 3.1.7 `PackageReference`, so the + source generator runs, and its assembly carries `[GeneratedAssemblyMessagePackResolverAttribute( + typeof(GeneratedMessagePackResolver), 3, 1)]`. For `CultGeometryDomainNode`, `DomainDocument` and + `FeatureClaim`, `StandardResolver` returns `GeneratedMessagePackResolver`'s formatter and every public + singleton returns a different instance or null. For `CultGeometryDomainNode[]` and `FeatureClaim[]` it is + the generated resolver's `ArrayFormatter`, not `DynamicGenericResolver`'s. No Geometry id-path type matches + a family singleton. Caching.Tests has no generator (analyzers do not flow through a `ProjectReference`), so + its keyed types match `DynamicObjectResolver`. +- Generated assemblies (Soul s5, fresh generator-enabled assembly `SoulGen`, `job2.log` GEN rows): a keyed + `List` or `Dictionary` subclass (`GKColl`, `GKDict`) is written by key, wire `["c",[7]]`, with its elements + dropped by MessagePack; the walk reads it by key too (Plan 0). A generated-assembly `[DataContract]` + enumerable (`GDcSeq`) is still written by `DynamicObjectResolver`, because the generator emits no + DataContract formatters. +- Null standard (Soul s6 finding `null-standard-untyped`): for a non-generic interface or abstract class that + enumerates arms, `StandardResolver` returns null instead of throwing, and the door neither refuses nor + types it. +- Wire pin (Soul s5, s6 `SoulS5Pin`): a deterministic document with 12 minted ids, encoded by the codec and + decoded independently with `MessagePackSerializer.ConvertToJson`, carries every id. The ids are + byte-identical at `494ad9ab` and `57fdf70c`, `0f8964b58a08` and `7eb4fb13d530` among them. +- Not measured: Unity/IL2CPP, where the package bundles its own `MessagePack.dll` (version not read) and the + dynamic resolvers cannot emit, and MessagePack versions other than 3.1.7. + +**Whole-list holders at `57fdf70c`** (Imagination r2 for `c2a-id-closed-world`, 2026-10-16; probe tests in a +scratch commit `dc629459` that was never pushed, Yggdrasil `sdk:10.0`; logs `job1.log` and `job2.log` in the +Imagination scratchpad `imagination-c2a-closed-world-r2`). +- Shapes the walk does not own already behave as whole values. A keyed document holding + `ImmutableArray`, `(List, int)`, `KeyValuePair>`, a non-enumerable + `[DataContract]` holder, or a string-keyed holder (`[Key("items")]`) gives `Inspect` problems 0, an empty + IdPath, `CanHoldIds` false, `Plan` 0 (random and deterministic), and `Holds` false with ids set. +- The marker and the copy check do not count them. Committed with ids `aaaaaaaaaaaa`/`bbbbbbbbbbbb`, the store + header stays `cultcache.store.v1`. A fresh cache that writes another record over that file (the copy-forward + check) keeps `v1`. Reload returns the ids exactly as written, and unset ids stay null after write and reload. +- Variants replace them whole, and that is the C2a resolver's behaviour for every list. A variant `Set` of the + `ImmutableArray` slot resolved to `[9:eeeeeeeeeeee]` with the base untouched, header `v2`, `HoldsIds` false. + A path step `[1:aaaaaaaaaaaa]` is refused: "this runtime resolves top-level Set overrides only". No C2a gap. + The gap belongs to C2b, in the cut sequence below. +- AllowPrivate is read from the most derived `[MessagePackObject]`. `ApDerived` (AllowPrivate over a base + without it) is written `[[],[],4,[]]`: base-chain privates and a private-getter property, with + `ObjectSerializationInfo` at `allowPrivate: true` = `[baseStash, own, N, Pg]`. `NoApDerived` (plain over an + AllowPrivate base) is written `[nil, 5]`. The walk read only `[N]` for both. +- A `[DataContract]` `List` subclass is written by `NonGenericListFormatter`, and its record does not read + back ("System.Object[] is not of type PElem"), whatever the walk does. That is a MessagePack limit, not an + id fact. + ## 2. Model page (step 0b) | Kind | Named by | Over time | Decided by | @@ -99,16 +214,16 @@ Probes run 2026-10-12 from `F:\Projects\CultLib-c2afix` with git only (no builds | Base reference | the base's key, in the record's variant slot | changed only by an explicit rebase; a base cannot be deleted while it has variants, unless the same batch rebases or removes them | author; the cache refuses a cycle, a missing base, a different concrete type, a global type | | Override entry | `(op, path, id, value)`; op is Set, Insert or Remove | added by editing; removed by clearing; survives base edits while its path resolves | author; the cache refuses entries a base edit strands (Q2); an entry whose member left the type soft-drifts with a warning (Q4 c) | | Member path | a sequence of `(slot, elementId)` steps; `slot` is the MessagePack `[Key]`; `elementId` is `""` unless the step enters an object-list element | a type change can invalidate it | the type's owner (slot authority); ids by the element's creator | -| Element id | a `[CultElementId]` string member on an object-list element type; `""` means unset | minted by the cache on write when unset; copied unchanged into variants; a moved element is a new element | cache mints it; it is never retargeted | +| Element id | a `[CultElementId]` string member on the element type of an object list in a shape the walk owns (arrays, `IEnumerable`, `IDictionary`/`IReadOnlyDictionary` values, `[MessagePackObject]`/`[Union]` types with integer keys), one MessagePack writes and reads back (integer `[Key]`, no `[IgnoreMember]`; public, or any visibility under the most derived `AllowPrivate`); `""` means unset. **Exception (q3 amended 2026-10-16):** a shape the walk does not own is one whole value: `ImmutableArray`, `ArraySegment`, `Memory`, `KeyValuePair`, tuples, `Lazy`, unkeyed or `[DataContract]` classes, string-keyed types, a type or member with `[MessagePackFormatter]`, and a non-public member without `AllowPrivate`. Its elements carry no ids: a `[CultElementId]` inside is a plain string | minted by the cache on write when unset; copied unchanged into variants; a moved element is a new element. Inside a whole value: never minted, never checked, never counted for `v3` | cache mints it; it is never retargeted. `CultElementIds.Classify` alone decides owned versus whole | | Resolved view | `CultStoredDocument.Document` for a variant key | recomputed inside the hold that admits the variant, or any base on its chain | the cache only; never persisted, never written back (Q1) | | Index and name entries | as today, from the resolved view | a dependent's entries are dropped and re-added in the same `Apply` as its base change | the cache; a variant must override the `[CultName]` member (rule R6) | | Watches | as today; each re-resolved dependent emits `Updated` with its own `Sequence` | published in the same hold as the base change | the cache | | Compare-exchange | `(schemaId, storedAt)` of the stored delta (ruling 2) | a base change does not stamp its variants | the cache; the resolved view is not an identity | -| Persisted format version | store header `formatVersion` | `cultcache.store.v2` when the store holds at least one variant; `v1` otherwise (Q5) | the writing store, from content | +| Persisted format version | store header `formatVersion` | `cultcache.store.v2` when the store holds at least one variant; `v1` otherwise (Q5); a marked `v3` (`v5` directory) stays while a write copies any record forward, and only a write that encodes every record drops it by content | the writing store, from content; the durable header alone says the bytes hold element ids | | Single-file store file | the path; one MessagePack array `[header, catalog, records]` | replaced whole and atomically; refused when unreadable; see "Store authorities" | each runtime's one reader decides what is a store; the writer derives the header from it | | Store lock and staging files | `.lock`; a unique temp beside the store | the lock is taken by C# and Rust writers (CultMesh included, through `ReplaceDurable`); staging is removed on a failed write | the store, never CultMesh or a consumer | | Variant slot on the wire (CultNet) | **empty cell**: no CultNet message carries a delta today | C5 | ruling 4 decides deltas; the message shape is C5's design | -| Dictionary entries, scalar lists, arrays | **empty cell**: no stable identity exists | whole-member Set only (Q7) | Q7 | +| Dictionary entries, scalar lists, arrays, whole-list holders | **empty cell**: no stable identity exists | whole-member Set only (Q7; whole-list holders by q3 amended) | Q7, q3 amended | | Template (non-item) bases | **empty cell**: parked by ruling 6 | none | not in these cuts | ## 3. Where the Body contradicts the target @@ -220,20 +335,21 @@ Dictionary entries, scalar lists and arrays are replaced whole. - **Recommend yes.** Aetheria's cases (`Hardpoints` whole; `Behaviors[id]...Max`) fit. Keyed dictionary addressing can be added later without a format change. Affects C2. -## Cut sequence (2026-10-12) +## Cut sequence (2026-10-15) -Mapped now (specs in the mind): `c2a-declared-apply` r2, the C# commit-time fix batch on -`hands/variants-c2a` (the stash, finished, with tests and contract text), and its sibling -`main-into-c2a` r2, main merged into C2a on `hands/variants-c2a-main` in its own worktree. Later -cuts, one line each, mapped only when Self is about to dispatch them: +Landed since 2026-10-12: `c2a-declared-apply` (r3, Soul s5) and `main-into-c2a` (r2, Soul s2); the +reconcile step is done, `hands/variants-c2a` is at the reconciled head `6afa8ed9`. Element ids part 1, +`c2a-element-id-holes` (r6), is held by Soul s5/s6 on `hands/variants-c2a-element-id-holes`. Mapped +now: part 2, `c2a-id-closed-world` r2 (2026-10-16, under the amended q3), not blocked. Its open fork +`variants:question:c2a-whole-list-opt-in` holds under every option. Later cuts, one line each, mapped +only when Self is about to dispatch them: -1. **reconcile**: merge `hands/variants-c2a` into `hands/variants-c2a-main` after both Soul passes - hold; Self moves `hands/variants-c2a` there. Waits on both mapped cuts. +1. **reconcile**: done at `6afa8ed9`. 2. **parity-ts**: TypeScript write set, one record per key, undeclared records carried and refused, - unheld removals judged at commit. Waits on reconcile. + unheld removals judged at commit. Waits on element ids parts 1 and 2 merged into C2a. 3. **parity-rs**: Rust `push_all` and `put_prepared_batch` apply a write set, ownership by - declaration, the same commit-time checks. Waits on reconcile; can run beside parity-ts. -4. **parity-py**: Python write-set apply. Waits on reconcile and on `cultlib-gaps` cut + declaration, the same commit-time checks. Waits as parity-ts; can run beside it. +4. **parity-py**: Python write-set apply. Waits as parity-ts, and on `cultlib-gaps` cut `py-msgpack-default` landing on main (then a main merge into C2a first). 5. **write-set-vectors**: one cross-runtime manifest, read and acted on by all four runtimes; carries follow-ups `c0-bad-canonical-in-manifest`, `legacy-vectors-all-runtimes`, @@ -242,6 +358,11 @@ cuts, one line each, mapped only when Self is about to dispatch them: with an empty diff against the verified head; no tag. Waits on write-set-vectors, Soul holding, and `asura:cut_spec:cut-semver-measured` landed. Asura Cut 3 bases on its merge commit. +Element-id part 2: `c2a-id-closed-world` r2 decides deferred findings `s5.member-set-misses-allowprivate`, `s5.enumerating-rule-unpinned`, `s6.null-standard-untyped`, `s6.two-recognisers-one-rule`, `s4.unrecognised-holders-fail-open` and follow-up `c2a-datacontract-holder-unwalked` (see its `first`). + +- **marker-for-whole-lists** (only if `c2a-whole-list-opt-in` is ruled `marker-when-ids-declared` or `marker-for-any-keyed-object`): a CarriesElements predicate, a refusal unless the member is marked, and a public attribute. It is additive after part 2. +- **C2b constraint** (owed to `variants:follow_up:remaining-variant-cuts`): nested path resolution must refuse a step `[slot:elementId]` that enters a whole value, naming the holder. A whole value is any shape `CultElementIds.Classify` calls Leaf, so C2b asks Classify and keeps no recogniser of its own. Today's C2a resolver refuses every non-top-level step, so no store can hold such an override yet. + Rationale. The 2026-10-02 order (parity, vectors, then merge main) was right when main was close. Main has since moved 453 commits and rebuilt exactly what the parity cuts change: cultcache-ts payload ownership, cultmesh-ts, cultcache-rs. Parity mapped before the merge would be mapped @@ -1484,7 +1605,7 @@ The one door per runtime, and who walks through it: - **Inputs.** The bytes at the path. **Outputs.** The decoded store with header and catalog; or a typed refusal (`CultStoreUnreadableException` in C#; `CultCacheStoreUnreadable{path, kind}` in Rust, kind `Undecodable` or `UnsupportedFormat`; `StoreUnreadableError` in TS and Python). A refusal names a header only as `cultcache.store.v`. - **Derived state.** The write header is never decided by the writer: it is the reader's verdict (header read, whole-store flag, whether ids or variants are held). A rewrite of a file that cannot be read cannot happen, because the reader throws first. - **Forbidden writers.** A second header classifier. Deleted: Rust `leading_header`, `read_array_header`, `read_string`; cultnet-ts's own decode of the store header; the TS async `readStore`; Rust `StoreUnreadableError` (negative grep `StoreUnreadableError` in `packages/cultcache-rs`: no match); Python `_decode_v1_snapshot`. -- **Shared paths.** cultnet-ts reaches the header decision only through `readSingleFileStore` (AUTH:4, :321) and checks only record count, schema id and type. cultnet-ts therefore declares cultcache-ts `^0.15.0`, the first release that exports it. +- **Shared paths.** cultnet-ts reaches the header decision only through `readSingleFileStore` (AUTH:4, :321) and checks only record count, schema id and type. cultnet-ts therefore needs a cultcache-ts release that exports it, which is the release that ships with this change; the manifests move to that version then (c0-ships-with-c2a), not before. - **Cut line.** `merge-r1` h3 (Rust) and h4 (TS, cultnet-ts); spec `cut-merge-r1.r3` authority map. - **Pinned by.** `tests/vectors/document-variants-c2a/readability/manifest.txt`, walked by `StoreReadabilityTests.OpenReadsExactlyTheFilesTheVectorsSayItReads` (C#), `CultMeshSingleFileReadabilityTests.AWriteReplacesAFileExactlyWhenItReads` (Mesh), `a_file_is_replaced_exactly_when_it_reads` (RS:5132), "SingleFileMessagePackBackingStore replaces a file exactly when it reads" (TS test :2321) and `test_single_file_replaces_a_file_exactly_when_it_reads` (Python). AUTH cases: `packages/cultnet-ts/test/idunn-runtime-authority.test.ts`. Declared divergences in the manifest: `legacy-envelopes.bin` (C# refuses; the others read and replace it) and `variant-slot-v3.bin` (C# reads; the others refuse). - **Known gaps.** `variants:follow_up:c0-bad-canonical-in-manifest` (`current-catalog-bad-canonical.msgpack` is not in the manifest; TS reads it, Rust refuses). `variants:follow_up:legacy-vectors-all-runtimes` (the three `legacy-catalog-*` c0 vectors have a verdict only in C# tests; all four runtimes refuse them, probed). `variants:follow_up:range-check-in-ci` (`npm run test:pack:ts` builds every admitted tag and checks exports, but `.github/workflows/publish-packages.yml` does not run it). diff --git a/docs/document-variants-id-sweep.md b/docs/document-variants-id-sweep.md new file mode 100644 index 00000000..03fad9cb --- /dev/null +++ b/docs/document-variants-id-sweep.md @@ -0,0 +1,272 @@ +# Element-id sweep list (C2a) + +Data for the per-consumer sweep in docs/document-variants-cut.md, C2. Every object type below sits in a list (`List`, `T[]`, `IEnumerable`, and the like) reachable from a registered document type, and needs one string member with `[Key(n)]` and `[CultElementId]` before that consumer can take a CultLib that requires ids. A union base takes the member on the base; its subclasses inherit it, so put the slot above the subclasses' highest key. + +Method: a static scan of `F:\Projects\` C# files for types marked `[CultDocument(`, walked through keyed members, base classes, dictionary values and `[Union]` subtypes. Read-only; no consumer was edited. Worktree copies (`Aetheria-catalog`, `CultLib-*`, and so on) are excluded. Struct elements are values and need no id. Non-C# consumers carry the id as an ordinary member and are not listed. The scan is text-based: a document registered without a literal `[CultDocument(` attribute, or a type whose members lack an integer `[Key]`, is missed, so the sweep pass should confirm each repo by registering its types once. + +Lines: `Type file:line (via Owner.Member)`, paths relative to the consumer root. + + +## Aetheria (13 element types) + +- BehaviorData Assets/Scripts/ServerShared/Behaviors/Behaviors.cs:190 (via ConsumableItemData.Behaviors) +- EntityPack Assets/Scripts/ServerShared/EntitySerializer.cs:171 (via ZonePack.Entities) +- ProductRole Assets/Scripts/ServerShared/FactionProduct.cs:38 (via FactionProductData.Roles) +- InputLayoutRow Assets/Scripts/ServerShared/InputLayout.cs:32 (via InputLayout.Rows) +- InputLayoutColumn Assets/Scripts/ServerShared/InputLayout.cs:52 (via InputLayoutKeyRow.Columns) +- ItemRole Assets/Scripts/ServerShared/ItemData.cs:325 (via CraftedItemData.Roles) +- AudioStat Assets/Scripts/ServerShared/ItemData.cs:441 (via EquippableItemData.AudioStats) +- HardpointData Assets/Scripts/ServerShared/ItemData.cs:553 (via HullData.Hardpoints) +- StatTerm Assets/Scripts/ServerShared/ItemData.cs:630 (via PerformanceStat.Terms) +- LoadoutSlot Assets/Scripts/ServerShared/Loadout.cs:23 (via Loadout.Slots) +- RoleFill Assets/Scripts/ServerShared/Provenance.cs:84 (via Lot.Roles) +- SavedActionBarBinding Assets/Scripts/ServerShared/SavedGame.cs:168 (via SavedGame.ActionBarBindings) +- Asteroid Assets/Scripts/ServerShared/ZoneData.cs:133 (via AsteroidBeltData.Asteroids) + +## AetheriaEve (115 element types) + +- AetheriaCorporationAllegiance Aetheria.State/Documents/AetheriaCorporation.cs:67 (via AetheriaCorporation.Allegiances) +- AetheriaShapeCell Aetheria.State/Documents/AetheriaItemDefinition.cs:178 (via AetheriaItemDefinition.ShapeCells) +- AetheriaItemHardpoint Aetheria.State/Documents/AetheriaItemDefinition.cs:188 (via AetheriaItemDefinition.Hardpoints) +- AetheriaCurveKey Aetheria.State/Documents/AetheriaItemDefinition.cs:222 (via AetheriaItemDefinition.ThermalPerformanceCurveKeys) +- AetheriaItemAudioStat Aetheria.State/Documents/AetheriaItemDefinition.cs:238 (via AetheriaItemDefinition.AudioStats) +- AetheriaBehaviorPayload Aetheria.State/Documents/AetheriaItemDefinition.cs:267 (via AetheriaItemDefinition.BehaviorPayloads) +- AetheriaBehaviorField Aetheria.State/Documents/AetheriaItemDefinition.cs:286 (via AetheriaBehaviorPayload.Fields) +- AetheriaBehaviorMapEntry Aetheria.State/Documents/AetheriaItemDefinition.cs:296 (via AetheriaBehaviorValue.MapEntries) +- AetheriaBehaviorValue Aetheria.State/Documents/AetheriaItemDefinition.cs:306 (via AetheriaBehaviorValue.Children) +- AetheriaLegacyCatalogFile Aetheria.State/Documents/AetheriaLegacyCatalogQuarantine.cs:41 (via AetheriaLegacyCatalogQuarantine.NameFiles) +- AetheriaMigrationCount Aetheria.State/Documents/AetheriaMigrationLedger.cs:35 (via AetheriaMigrationLedger.Counts) +- AetheriaLegacyCatalogEntrySummary Aetheria.State/Documents/AetheriaMigrationLedger.cs:45 (via AetheriaMigrationLedger.LegacyCatalogEntries) +- AetheriaStoryFileHash Aetheria.State/Documents/AetheriaRuntimeStateDocuments.cs:42 (via AetheriaPlayerSettings.StoryFileHashes) +- AetheriaInputBindingOverride Aetheria.State/Documents/AetheriaRuntimeStateDocuments.cs:85 (via AetheriaPlayerInputSettings.BindingOverrides) +- AetheriaEntityLoadout Aetheria.State/Documents/AetheriaRuntimeStateDocuments.cs:119 (via AetheriaEntityLoadout.Children) +- AetheriaLoadoutItemSlot Aetheria.State/Documents/AetheriaRuntimeStateDocuments.cs:186 (via AetheriaEntityLoadout.Equipment) +- AetheriaCargoBayLoadout Aetheria.State/Documents/AetheriaRuntimeStateDocuments.cs:199 (via AetheriaEntityLoadout.CargoContents) +- AetheriaFactionZoneState Aetheria.State/Documents/AetheriaRuntimeStateDocuments.cs:268 (via AetheriaRunState.HomeZones) +- AetheriaAgentTaskState Aetheria.State/Documents/AetheriaRuntimeStateDocuments.cs:275 (via AetheriaRunState.AgentTasks) +- AetheriaDroppedPickupSnapshot Aetheria.State/Documents/AetheriaRuntimeStateDocuments.cs:346 (via AetheriaZoneState.DroppedPickups) +- AetheriaOrbitSnapshot Aetheria.State/Documents/AetheriaRuntimeStateDocuments.cs:362 (via AetheriaZoneState.Orbits) +- AetheriaBodySnapshot Aetheria.State/Documents/AetheriaRuntimeStateDocuments.cs:384 (via AetheriaZoneState.Bodies) +- AetheriaBodyResource Aetheria.State/Documents/AetheriaRuntimeStateDocuments.cs:449 (via AetheriaBodySnapshot.Resources) +- AetheriaAsteroidSnapshot Aetheria.State/Documents/AetheriaRuntimeStateDocuments.cs:459 (via AetheriaBodySnapshot.Asteroids) +- AetheriaAsteroidMiningAccumulatorSnapshot Aetheria.State/Documents/AetheriaRuntimeStateDocuments.cs:484 (via AetheriaAsteroidSnapshot.MiningAccumulators) +- AetheriaColor Aetheria.State/Documents/AetheriaRuntimeStateDocuments.cs:545 (via AetheriaGasGiantVisualState.Colors) +- AetheriaFogFieldEmitterSnapshot Aetheria.State/Documents/AetheriaRuntimeStateDocuments.cs:691 (via AetheriaEntitySnapshot.FogFieldEmitters) +- AetheriaLoadoutGenerationSelection Aetheria.State/Documents/AetheriaRuntimeStateDocuments.cs:723 (via AetheriaLoadoutGenerationReceipt.Selections) +- AetheriaEntityContactSnapshot Aetheria.State/Documents/AetheriaRuntimeStateDocuments.cs:734 (via AetheriaEntitySnapshot.Contacts) +- AetheriaEntityItemSlot Aetheria.State/Documents/AetheriaRuntimeStateDocuments.cs:751 (via AetheriaEntitySnapshot.Equipment) +- AetheriaWeaponGroupSnapshot Aetheria.State/Documents/AetheriaRuntimeStateDocuments.cs:782 (via AetheriaEntitySnapshot.WeaponGroups) +- AetheriaEntityStatGrid Aetheria.State/Documents/AetheriaRuntimeStateDocuments.cs:789 (via AetheriaEntitySnapshot.StatGrids) +- AetheriaActiveConsumableSnapshot Aetheria.State/Documents/AetheriaRuntimeStateDocuments.cs:805 (via AetheriaEntitySnapshot.ActiveConsumables) +- AetheriaConsumableBehaviorStateSnapshot Aetheria.State/Documents/AetheriaRuntimeStateDocuments.cs:828 (via AetheriaActiveConsumableSnapshot.BehaviorStates) +- AetheriaBehaviorProgressSnapshot Aetheria.State/Documents/AetheriaRuntimeStateDocuments.cs:844 (via AetheriaEntitySnapshot.BehaviorProgress) +- AetheriaWeaponStateSnapshot Aetheria.State/Documents/AetheriaRuntimeStateDocuments.cs:863 (via AetheriaEntitySnapshot.WeaponStates) +- AetheriaBehaviorStateSnapshot Aetheria.State/Documents/AetheriaRuntimeStateDocuments.cs:924 (via AetheriaEntitySnapshot.BehaviorStates) +- AetheriaFactionRelationshipState Aetheria.State/Documents/AetheriaRuntimeStateDocuments.cs:1093 (via AetheriaRunState.FactionRelationships) +- AetheriaItemRarityTier Aetheria.State/Documents/AetheriaTradeValuePolicy.cs:29 (via AetheriaTradeValuePolicy.Tiers) +- AetheriaRuntimeAgentTaskCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeAgentTasks.cs:35 (via AetheriaRuntimeRunCheckpointCommit.AgentTasks) +- AetheriaRuntimeAssetManifestEntry Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeAssetDocuments.cs:102 (via AetheriaRuntimeAssetManifestDocument.Assets) +- AetheriaRuntimeCurveKey Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeCatalogSnapshot.cs:446 (via AetheriaRuntimeShotReceiptCommit.GuidanceCurve) +- AetheriaRuntimeCorporation Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeCatalogSnapshot.cs:620 (via AetheriaRuntimeCatalogSnapshot.Corporations) +- AetheriaRuntimeNameFile Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeCatalogSnapshot.cs:669 (via AetheriaRuntimeCatalogSnapshot.NameFiles) +- AetheriaRuntimeLoadoutTemplateSnapshot Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeCatalogSnapshot.cs:796 (via AetheriaRuntimeLoadoutTemplatesDocument.Templates) +- AetheriaRuntimeDaemonCommandBoundaryEntry Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeDaemonDocuments.cs:462 (via AetheriaRuntimeDaemonCommandBoundaryDocument.Commands) +- AetheriaRuntimeDaemonSoaIdentityDocument Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeDaemonSoaDocuments.cs:143 (via AetheriaRuntimeDaemonSoaViewDocument.Identities) +- AetheriaRuntimeDaemonRenderGroupDocument Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeDaemonSoaDocuments.cs:156 (via AetheriaRuntimeDaemonSoaViewDocument.RenderGroups) +- AetheriaRuntimeDaemonSoaBufferDocument Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeDaemonSoaDocuments.cs:222 (via AetheriaRuntimeDaemonSoaViewDocument.Buffers) +- AetheriaRuntimeDaemonSoaColumnDocument Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeDaemonSoaDocuments.cs:253 (via AetheriaRuntimeDaemonSoaViewDocument.Columns) +- AetheriaRuntimeDaemonSoaDirtyRangeDocument Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeDaemonSoaDocuments.cs:284 (via AetheriaRuntimeDaemonSoaViewDocument.DirtyRanges) +- AetheriaRuntimeEveSurfaceAdvertisement Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeEveSurfaceCatalog.cs:11 (via AetheriaRuntimeDaemonProviderAdvertisementDocument.EveSurfaces) +- AetheriaRuntimeEvePluginRequirement Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeEveSurfaceCatalog.cs:58 (via AetheriaRuntimeEveSurfaceAdvertisement.RequiresPlugins) +- AetheriaRuntimeZoneTargetRow Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeGameViewportDocuments.cs:410 (via AetheriaRuntimeZoneContactsDocument.Targets) +- AetheriaRuntimeZoneContactRow Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeGameViewportDocuments.cs:441 (via AetheriaRuntimeZoneContactsDocument.Contacts) +- AetheriaRuntimeStationStockItem Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeGameViewportDocuments.cs:551 (via AetheriaRuntimeStationRefitEntityOption.CargoItems) +- AetheriaRuntimeStationRefitEntityOption Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeGameViewportDocuments.cs:585 (via AetheriaRuntimeStationRefitDocument.AvailableEntities) +- AetheriaRuntimeStationDockingBayRow Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeGameViewportDocuments.cs:620 (via AetheriaRuntimeStationRefitDocument.DockingBays) +- AetheriaRuntimeStationLoadoutRestoreOption Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeGameViewportDocuments.cs:655 (via AetheriaRuntimeStationRefitDocument.LoadoutRestoreOptions) +- AetheriaRuntimeStationCargoTargetRow Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeGameViewportDocuments.cs:674 (via AetheriaRuntimeStationRefitDocument.CargoTargets) +- AetheriaRuntimeSectorMapZone Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeGameViewportDocuments.cs:765 (via AetheriaRuntimeSectorMapDocument.Zones) +- AetheriaRuntimeSectorMapLink Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeGameViewportDocuments.cs:802 (via AetheriaRuntimeSectorMapDocument.Links) +- AetheriaRuntimeZoneRenderAdjacentZone Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeGameViewportDocuments.cs:1197 (via AetheriaRuntimeZoneRenderDocument.AdjacentZones) +- AetheriaRuntimeZoneRenderWormholeExit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeGameViewportDocuments.cs:1210 (via AetheriaRuntimeZoneRenderDocument.WormholeExits) +- AetheriaRuntimeZoneRenderBodyPose Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeGameViewportDocuments.cs:1229 (via AetheriaRuntimeZoneRenderDocument.BodyPoses) +- AetheriaRuntimeZoneRenderAsteroidBeltPose Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeGameViewportDocuments.cs:1260 (via AetheriaRuntimeZoneRenderDocument.AsteroidBeltPoses) +- AetheriaRuntimeZoneRenderAsteroidInstancePose Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeGameViewportDocuments.cs:1286 (via AetheriaRuntimeZoneRenderAsteroidBeltPose.InstancePoses) +- AetheriaRuntimeViewportObject Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeGameViewportDocuments.cs:1382 (via AetheriaRuntimeGameViewportDocument.Objects) +- AetheriaRuntimeInventoryItem Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeGameViewportDocuments.cs:1458 (via AetheriaRuntimeViewportObject.Inventory) +- AetheriaRuntimeGravityInfluence Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeGameViewportDocuments.cs:1493 (via AetheriaRuntimeGameViewportDocument.GravityInfluences) +- AetheriaRuntimeBodyView Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeGameViewportDocuments.cs:1530 (via AetheriaRuntimeGameViewportDocument.Bodies) +- AetheriaHangarShip Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeHangarDocuments.cs:119 (via AetheriaHangarState.Ships) +- AetheriaHangarItemStack Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeHangarDocuments.cs:129 (via AetheriaHangarState.Inventory) +- AetheriaHangarCurrency Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeHangarDocuments.cs:136 (via AetheriaHangarState.Currencies) +- AetheriaDeploymentReceipt Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeHangarDocuments.cs:155 (via AetheriaHangarState.Deployments) +- AetheriaProgressionVerseOption Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeProgressionSourceDocuments.cs:45 (via AetheriaProgressionSourceDocument.AvailableVerses) +- AetheriaRuntimeStoryFileHashDocument Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSettingsDocuments.cs:87 (via AetheriaRuntimePlayerSettingsDocument.StoryFileHashes) +- AetheriaRuntimeInputBindingOverrideDocument Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSettingsDocuments.cs:108 (via AetheriaRuntimePlayerSettingsDocument.BindingOverrides) +- AetheriaRuntimeEntityLoadoutCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:78 (via AetheriaRuntimeEntityLoadoutCommit.Children) +- AetheriaRuntimeLoadoutItemSlotCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:143 (via AetheriaRuntimeEntitySnapshotCommit.Equipment) +- AetheriaRuntimeCargoBayLoadoutCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:159 (via AetheriaRuntimeEntitySnapshotCommit.CargoContents) +- AetheriaRuntimeGameEventCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:490 (via AetheriaRuntimeRunCheckpointCommit.GameEvents) +- AetheriaRuntimeFactionZoneCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:509 (via AetheriaRuntimeRunCheckpointCommit.HomeZones) +- AetheriaRuntimeShotReceiptCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:539 (via AetheriaRuntimeRunCheckpointCommit.ShotReceipts) +- AetheriaRuntimePickupContactReceiptCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:591 (via AetheriaRuntimeRunCheckpointCommit.PickupContactReceipts) +- AetheriaRuntimeDamageCellCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:603 (via AetheriaRuntimeShotReceiptCommit.DamageCells) +- AetheriaRuntimeCorporationSurveyCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:614 (via AetheriaRuntimeRunCheckpointCommit.CorporationSurveys) +- AetheriaRuntimeFactionRelationshipCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:623 (via AetheriaRuntimeRunCheckpointCommit.FactionRelationships) +- AetheriaRuntimeZoneSnapshotCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:636 (via AetheriaRuntimeRunCheckpointCommit.Zones) +- AetheriaRuntimePhysicalPayloadCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:699 (via AetheriaRuntimeZoneSnapshotCommit.PhysicalPayloads) +- AetheriaRuntimeDroppedPickupCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:771 (via AetheriaRuntimeZoneSnapshotCommit.DroppedPickups) +- AetheriaRuntimeOrbitSnapshotCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:805 (via AetheriaRuntimeZoneSnapshotCommit.Orbits) +- AetheriaRuntimeBodySnapshotCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:830 (via AetheriaRuntimeZoneSnapshotCommit.Bodies) +- AetheriaRuntimeBodyResourceCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:901 (via AetheriaRuntimeBodySnapshotCommit.Resources) +- AetheriaRuntimeAsteroidCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:911 (via AetheriaRuntimeBodySnapshotCommit.Asteroids) +- AetheriaRuntimeAsteroidMiningAccumulatorCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:936 (via AetheriaRuntimeAsteroidCommit.MiningAccumulators) +- AetheriaRuntimeColorCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:1008 (via AetheriaRuntimeGasGiantVisualCommit.Colors) +- AetheriaRuntimeEntitySnapshotCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:1024 (via AetheriaRuntimeZoneSnapshotCommit.Entities) +- AetheriaRuntimeFogFieldEmitterCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:1198 (via AetheriaRuntimeEntitySnapshotCommit.FogFieldEmitters) +- AetheriaRuntimeEquipmentStateCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:1240 (via AetheriaRuntimeEntitySnapshotCommit.EquipmentStates) +- AetheriaRuntimeLoadoutGenerationSelectionCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:1264 (via AetheriaRuntimeLoadoutGenerationReceiptCommit.Selections) +- AetheriaRuntimeEntityContactCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:1275 (via AetheriaRuntimeEntitySnapshotCommit.Contacts) +- AetheriaRuntimeEntityStatGridCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:1291 (via AetheriaRuntimeEntitySnapshotCommit.StatGrids) +- AetheriaRuntimeActiveConsumableCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:1307 (via AetheriaRuntimeEntitySnapshotCommit.ActiveConsumables) +- AetheriaRuntimeConsumableBehaviorStateCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:1330 (via AetheriaRuntimeActiveConsumableCommit.BehaviorStates) +- AetheriaRuntimeBehaviorProgressCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:1355 (via AetheriaRuntimeEntitySnapshotCommit.BehaviorProgress) +- AetheriaRuntimeWeaponStateCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:1374 (via AetheriaRuntimeEntitySnapshotCommit.WeaponStates) +- AetheriaRuntimeBehaviorStateCommit Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeSnapshotDocuments.cs:1444 (via AetheriaRuntimeEntitySnapshotCommit.BehaviorStates) +- AetheriaRuntimeStarbridgeStationStockItem Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeStarbridgeDocuments.cs:71 (via AetheriaRuntimeStarbridgeScenarioDocument.StationStock) +- AetheriaRuntimeStarbridgeWaveDefinition Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeStarbridgeDocuments.cs:83 (via AetheriaRuntimeStarbridgeScenarioDocument.Waves) +- AetheriaRuntimeStarbridgeWaveForecast Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeStarbridgeDocuments.cs:94 (via AetheriaRuntimeStarbridgeSessionSummaryDocument.WaveForecast) +- AetheriaRuntimeStarbridgeRuntimeRole Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeStarbridgeDocuments.cs:104 (via AetheriaRuntimeStarbridgeScenarioDocument.RuntimeRoles) +- AetheriaRuntimeAuthorityRuntimeRole Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeVerseAuthorityPolicy.cs:194 (via AetheriaRuntimeVerseAuthorityPolicyDocument.RuntimeRoles) +- AetheriaRuntimeAuthorityRule Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeVerseAuthorityPolicy.cs:201 (via AetheriaRuntimeVerseAuthorityPolicyDocument.Rules) +- AetheriaRuntimeArenaSeat Packages/org.gamecult.aetheria.state/Runtime/AetheriaRuntimeVerseAuthorityPolicy.cs:255 (via AetheriaRuntimeArenaRosterDocument.Seats) + +## AquaSynth (0 element types) + +None. + + +## Aquarium (1 element types) + +- AquariumFaceVoiceEndpointState src/Aquarium.Epiphany/State/AquariumLiveState.cs:61 (via AquariumLiveState.FaceVoiceEndpoints) + +## Brokkr (4 element types) + +- BrokkrGameObjectSnapshot surfaces/unity/Packages/com.gamecult.brokkr/Runtime/BrokkrHostSnapshot.cs:42 (via BrokkrHostSnapshot.sceneObjects) +- BrokkrComponentSnapshot surfaces/unity/Packages/com.gamecult.brokkr/Runtime/BrokkrHostSnapshot.cs:62 (via BrokkrGameObjectSnapshot.components) +- BrokkrSerializedPropertySnapshot surfaces/unity/Packages/com.gamecult.brokkr/Runtime/BrokkrHostSnapshot.cs:73 (via BrokkrComponentSnapshot.properties) +- BrokkrAssetSnapshot surfaces/unity/Packages/com.gamecult.brokkr/Runtime/BrokkrHostSnapshot.cs:84 (via BrokkrHostSnapshot.assets) + +## Delvehold (0 element types) + +None. + + +## Eve (17 element types) + +- EveAssetCatalogEntry packages/org.gamecult.eve.surface/Runtime/EveAssetCatalogDocument.cs:52 (via EveAssetCatalogDocument.Assets) +- EveAssetVariant packages/org.gamecult.eve.surface/Runtime/EveAssetCatalogDocument.cs:89 (via EveAssetCatalogEntry.Variants) +- EveEntitySoaBuffer packages/org.gamecult.eve.surface/Runtime/EveEntitySoaViewDocument.cs:33 (via EveEntitySoaViewDocument.Buffers) +- EveEntitySoaColumn packages/org.gamecult.eve.surface/Runtime/EveEntitySoaViewDocument.cs:41 (via EveEntitySoaViewDocument.Columns) +- EveEntitySoaDirtyRange packages/org.gamecult.eve.surface/Runtime/EveEntitySoaViewDocument.cs:55 (via EveEntitySoaViewDocument.DirtyRanges) +- EveEntityIdentity packages/org.gamecult.eve.surface/Runtime/EveEntitySoaViewDocument.cs:64 (via EveEntitySoaViewDocument.Identities) +- EveEntityRenderGroup packages/org.gamecult.eve.surface/Runtime/EveEntitySoaViewDocument.cs:77 (via EveEntitySoaViewDocument.RenderGroups) +- EveInputActionDocument packages/org.gamecult.eve.surface/Runtime/EveInputCapabilityDocument.cs:22 (via EveInputCapabilityDocument.Actions) +- EveInputProfileDocument packages/org.gamecult.eve.surface/Runtime/EveInputCapabilityDocument.cs:51 (via EveInputCapabilityDocument.DefaultProfiles) +- EveInputBindingDocument packages/org.gamecult.eve.surface/Runtime/EveInputCapabilityDocument.cs:59 (via EveInputProfileDocument.Bindings) +- EveProviderWitness packages/org.gamecult.eve.surface/Runtime/EveProviderDocuments.cs:249 (via EveProviderAdvertisementDocument.Witnesses) +- EveAdvertisedSurface packages/org.gamecult.eve.surface/Runtime/EveProviderDocuments.cs:265 (via EveProviderAdvertisementDocument.Surfaces) +- EveAdvertisedCommand packages/org.gamecult.eve.surface/Runtime/EveProviderDocuments.cs:333 (via EveProviderAdvertisementDocument.Commands) +- EveSurfaceComponent packages/org.gamecult.eve.surface/Runtime/EveSurfaceDocument.cs:106 (via EveSurfaceComponent.Children) +- EveEmbeddedDocumentSlot packages/org.gamecult.eve.surface/Runtime/EveSurfaceDocument.cs:230 (via EveSurfaceComponent.EmbeddedDocuments) +- EveStyleToken packages/org.gamecult.eve.surface/Runtime/EveSurfaceDocument.cs:277 (via EveSurfaceTree.Styles) +- EveCommandTemplate packages/org.gamecult.eve.surface/Runtime/EveSurfaceDocument.cs:295 (via EveSurfaceDocument.Commands) + +## EvePlugins (1 element types) + +- EveFieldsSplatLayer plugins/eve-plugin-fields/unity/org.gamecult.eve.plugin-fields/Runtime/EveFieldsDocuments.cs:38 (via EveFieldsSplatsDocument.Layers) + +## Ghostlight (17 element types) + +These types are a vendored copy of Eve (`vendor/eve`); they change when Eve does and the pin bumps. + +- EveAssetCatalogEntry vendor/eve/packages/org.gamecult.eve.surface/Runtime/EveAssetCatalogDocument.cs:52 (via EveAssetCatalogDocument.Assets) +- EveAssetVariant vendor/eve/packages/org.gamecult.eve.surface/Runtime/EveAssetCatalogDocument.cs:89 (via EveAssetCatalogEntry.Variants) +- EveEntitySoaBuffer vendor/eve/packages/org.gamecult.eve.surface/Runtime/EveEntitySoaViewDocument.cs:33 (via EveEntitySoaViewDocument.Buffers) +- EveEntitySoaColumn vendor/eve/packages/org.gamecult.eve.surface/Runtime/EveEntitySoaViewDocument.cs:41 (via EveEntitySoaViewDocument.Columns) +- EveEntitySoaDirtyRange vendor/eve/packages/org.gamecult.eve.surface/Runtime/EveEntitySoaViewDocument.cs:55 (via EveEntitySoaViewDocument.DirtyRanges) +- EveEntityIdentity vendor/eve/packages/org.gamecult.eve.surface/Runtime/EveEntitySoaViewDocument.cs:64 (via EveEntitySoaViewDocument.Identities) +- EveEntityRenderGroup vendor/eve/packages/org.gamecult.eve.surface/Runtime/EveEntitySoaViewDocument.cs:77 (via EveEntitySoaViewDocument.RenderGroups) +- EveInputActionDocument vendor/eve/packages/org.gamecult.eve.surface/Runtime/EveInputCapabilityDocument.cs:22 (via EveInputCapabilityDocument.Actions) +- EveInputProfileDocument vendor/eve/packages/org.gamecult.eve.surface/Runtime/EveInputCapabilityDocument.cs:51 (via EveInputCapabilityDocument.DefaultProfiles) +- EveInputBindingDocument vendor/eve/packages/org.gamecult.eve.surface/Runtime/EveInputCapabilityDocument.cs:59 (via EveInputProfileDocument.Bindings) +- EveProviderWitness vendor/eve/packages/org.gamecult.eve.surface/Runtime/EveProviderDocuments.cs:249 (via EveProviderAdvertisementDocument.Witnesses) +- EveAdvertisedSurface vendor/eve/packages/org.gamecult.eve.surface/Runtime/EveProviderDocuments.cs:265 (via EveProviderAdvertisementDocument.Surfaces) +- EveAdvertisedCommand vendor/eve/packages/org.gamecult.eve.surface/Runtime/EveProviderDocuments.cs:333 (via EveProviderAdvertisementDocument.Commands) +- EveSurfaceComponent vendor/eve/packages/org.gamecult.eve.surface/Runtime/EveSurfaceDocument.cs:106 (via EveSurfaceComponent.Children) +- EveEmbeddedDocumentSlot vendor/eve/packages/org.gamecult.eve.surface/Runtime/EveSurfaceDocument.cs:230 (via EveSurfaceComponent.EmbeddedDocuments) +- EveStyleToken vendor/eve/packages/org.gamecult.eve.surface/Runtime/EveSurfaceDocument.cs:277 (via EveSurfaceTree.Styles) +- EveCommandTemplate vendor/eve/packages/org.gamecult.eve.surface/Runtime/EveSurfaceDocument.cs:295 (via EveSurfaceDocument.Commands) + +## Gjallar (3 element types) + +- GjallarAdvertisedSchema src/Gjallar/VerseState.cs:462 (via GjallarProviderAdvertisementRecord.Schemas) +- GjallarAdvertisedWitness src/Gjallar/VerseState.cs:470 (via GjallarProviderAdvertisementRecord.Witnesses) +- GjallarAdvertisedSurface src/Gjallar/VerseState.cs:480 (via GjallarProviderAdvertisementRecord.Surfaces) + +## Heimdall (12 element types) + +These types are a vendored copy of CultLib (`vendor/CultLib`); C2a already adds the ids in CultLib, so they resolve when the pin bumps. The witness bundle types are the fork reported with C2a. + +- CultGeometryDomainNode vendor/CultLib/src/GameCult.Geometry/CultGeometryDocuments.cs:61 (via CultGeometryDomainNode.Children) +- CultGeometryFeatureClaim vendor/CultLib/src/GameCult.Geometry/CultGeometryDocuments.cs:106 (via CultGeometryDomainNode.Claims) +- CultGeometryContributionRow vendor/CultLib/src/GameCult.Geometry/CultGeometryDocuments.cs:262 (via CultGeometrySelectedCutManifest.Diagnostics) +- CultMeshBodyDescriptor vendor/CultLib/src/GameCult.Mesh/CultMeshBodies.cs:35 (via CultMeshBodyPublicationDocument.Representations) +- CultMeshCdnChunkRef vendor/CultLib/src/GameCult.Mesh/CultMeshCdn.cs:73 (via CultMeshCdnArtifactManifest.Chunks) +- CultMeshDiscoveryCandidateDocument vendor/CultLib/src/GameCult.Mesh/CultMeshDiscoveryService.cs:418 (via CultMeshDiscoveryStateDocument.Candidates) +- CultMeshEntityPrefabComponent vendor/CultLib/src/GameCult.Mesh/CultMeshEntityPrefabs.cs:81 (via CultMeshEntityPrefabNode.Components) +- CultMeshEntityPrefabAssetRef vendor/CultLib/src/GameCult.Mesh/CultMeshEntityPrefabs.cs:96 (via CultMeshEntityPrefabPackage.Assets) +- CultMeshEntityPrefabNode vendor/CultLib/src/GameCult.Mesh/CultMeshEntityPrefabs.cs:127 (via CultMeshEntityPrefabPackage.Nodes) +- CultWitnessContractPin vendor/CultLib/src/GameCult.Networking/CultWitnessArtifactBundle.cs:105 (via CultWitnessArtifactBundle.Contracts) +- CultWitnessArtifactEntry vendor/CultLib/src/GameCult.Networking/CultWitnessArtifactBundle.cs:136 (via CultWitnessArtifactBundle.Artifacts) +- CultWitnessTimingEntry vendor/CultLib/src/GameCult.Networking/CultWitnessArtifactBundle.cs:179 (via CultWitnessArtifactBundle.TimingWitnesses) + +## Mimir (0 element types) + +None. + + +## StreamPixels (20 element types) + +`CultGeometryDomainNode` is the vendored CultLib copy; C2a already adds its id. + +- ActivationCondition apps/overlay-unity/Assets/Scripts/Assembly-CSharp/ActivationCondition.cs:18 (via CharacterConditions.Conditions) +- BattleEffect apps/overlay-unity/Assets/Scripts/Assembly-CSharp/BattleEffect.cs:8 (via BattleAction.Effects) +- GearStatEffect apps/overlay-unity/Assets/Scripts/Assembly-CSharp/GearStatEffect.cs:7 (via GearData.StatEffects) +- OverlaySceneIntent apps/overlay-unity/Assets/Scripts/Assembly-CSharp/ServiceLink/OverlayFrameDocuments.cs:33 (via OverlaySceneIntentWindow.Intents) +- PlayedIntentRecord apps/overlay-unity/Assets/Scripts/Assembly-CSharp/ServiceLink/PlayedIntentLedger.cs:25 (via OverlayPlayedIntents.Entries) +- StatModifier apps/overlay-unity/Assets/Scripts/Assembly-CSharp/StatModifier.cs:7 (via GearStatEffect.Modifiers) +- StatusEffect apps/overlay-unity/Assets/Scripts/Assembly-CSharp/StatusEffect.cs:10 (via StatusData.Effects) +- WeaponDamageTypeMultiplier apps/overlay-unity/Assets/Scripts/Assembly-CSharp/WeaponDamageTypeMultiplier.cs:7 (via WeaponData.DamageMultipliers) +- CultGeometryDomainNode vendor/CultLib/src/GameCult.Geometry/CultGeometryDocuments.cs:61 (via CultGeometryDomainNode.Children) +- CultGeometryFeatureClaim vendor/CultLib/src/GameCult.Geometry/CultGeometryDocuments.cs:106 (via CultGeometryDomainNode.Claims) +- CultGeometryContributionRow vendor/CultLib/src/GameCult.Geometry/CultGeometryDocuments.cs:262 (via CultGeometrySelectedCutManifest.Diagnostics) +- CultMeshBodyDescriptor vendor/CultLib/src/GameCult.Mesh/CultMeshBodies.cs:35 (via CultMeshBodyPublicationDocument.Representations) +- CultMeshCdnChunkRef vendor/CultLib/src/GameCult.Mesh/CultMeshCdn.cs:73 (via CultMeshCdnArtifactManifest.Chunks) +- CultMeshDiscoveryCandidateDocument vendor/CultLib/src/GameCult.Mesh/CultMeshDiscoveryService.cs:418 (via CultMeshDiscoveryStateDocument.Candidates) +- CultMeshEntityPrefabComponent vendor/CultLib/src/GameCult.Mesh/CultMeshEntityPrefabs.cs:81 (via CultMeshEntityPrefabNode.Components) +- CultMeshEntityPrefabAssetRef vendor/CultLib/src/GameCult.Mesh/CultMeshEntityPrefabs.cs:96 (via CultMeshEntityPrefabPackage.Assets) +- CultMeshEntityPrefabNode vendor/CultLib/src/GameCult.Mesh/CultMeshEntityPrefabs.cs:127 (via CultMeshEntityPrefabPackage.Nodes) +- CultWitnessContractPin vendor/CultLib/src/GameCult.Networking/CultWitnessArtifactBundle.cs:105 (via CultWitnessArtifactBundle.Contracts) +- CultWitnessArtifactEntry vendor/CultLib/src/GameCult.Networking/CultWitnessArtifactBundle.cs:136 (via CultWitnessArtifactBundle.Artifacts) +- CultWitnessTimingEntry vendor/CultLib/src/GameCult.Networking/CultWitnessArtifactBundle.cs:179 (via CultWitnessArtifactBundle.TimingWitnesses) + +## Ymir (1 element types) + +- YmirTransportLoweringDocument src/Ymir.Core/YmirServicePublicationDocument.cs:63 (via YmirProviderAdvertisementDocument.CommandLowerings) + +## CultLib itself + +C2a adds ids to: `CultGeometryDomainNode`, `CultGeometryFeatureClaim`, `CultGeometryContributionRow` (`src/GameCult.Geometry/CultGeometryDocuments.cs`); `CultMeshBodyDescriptor` (`src/GameCult.Mesh/CultMeshBodies.cs:35`); `CultMeshDiscoveryCandidateDocument` (`CultMeshDiscoveryService.cs:418`); `CultMeshEntityPrefabAssetRef`, `CultMeshEntityPrefabComponent`, `CultMeshEntityPrefabNode` (`CultMeshEntityPrefabs.cs`); `CultWitnessContractPin`, `CultWitnessArtifactEntry`, `CultWitnessTimingEntry` (`src/GameCult.Networking/CultWitnessArtifactBundle.cs`). + +`CultMeshCdnChunkRef` (`src/GameCult.Mesh/CultMeshCdn.cs:73`) has a derived id, `[CultElementId(nameof(Offset))]`: the decimal text of its offset, never random, so packing the same body encodes the same manifest bytes. `contracts/cultmesh/content-vectors.*.json` and the Rust runtime carry it. A consumer with content-addressed elements declares the same way. diff --git a/docs/research/2026-10-22-consumer-pinning-at-breaking-land.md b/docs/research/2026-10-22-consumer-pinning-at-breaking-land.md new file mode 100644 index 00000000..f4c4d1cc --- /dev/null +++ b/docs/research/2026-10-22-consumer-pinning-at-breaking-land.md @@ -0,0 +1,59 @@ +# Consumer pinning at a breaking land: prior art and internal state + +Serves fork `variants:question:unpinned-consumers-at-land` (campaign `variants`, session `self-2026-10-22-eureka-c2a`). +Eyes pass, 2026-10-22. Facts and evidence pointers only; no recommendation. + +Question as admitted: Eve.Surface and Brokkr build against an unpinned sibling `..\CultLib` source checkout. C2a (refuse element types with no id) lands on main. Options: pin-then-land, sweep-then-land, land-and-break. + +## Sources + +| # | Source | URL | Status | +|---|---|---|---| +| S1 | Abseil, About / philosophy | https://abseil.io/about/philosophy | fetched | +| S2 | Abseil, compatibility guidelines | https://abseil.io/about/compatibility | fetched | +| S3 | Software Engineering at Google, ch. 21 Dependency Management | https://abseil.io/resources/swe-book/html/ch21.html | fetched | +| S4 | Software Engineering at Google, ch. 16 Version Control and Branch Management | https://abseil.io/resources/swe-book/html/ch16.html | fetched (does not cover live at head) | +| S5 | Russ Cox, Minimal Version Selection | https://research.swtch.com/vgo-mvs | fetched | +| S6 | Cargo book, Cargo.toml vs Cargo.lock | https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html | fetched | +| S7 | npm docs, package-lock.json | https://docs.npmjs.com/cli/v10/configuring-npm/package-lock-json | fetched | +| S8 | Chromium src, dependencies doc (DEPS, gclient, roll-dep) | https://chromium.googlesource.com/chromium/src/+/main/docs/dependencies.md | fetched | +| S9 | Pro Git, Submodules | https://git-scm.com/book/en/v2/Git-Tools-Submodules | fetched | +| S10 | chromium infra dependencies.md | https://chromium.googlesource.com/infra/infra/+/main/docs/dependencies.md | 404, not retrieved | +| L1 | `F:\Projects\Aetheria\Directory.Build.targets`, `Directory.Build.props`, git history | local files | read | +| L2 | `F:\Projects\Aetheria\docs\cultcache-migration-cut.md` (Q6 table, lines ~565-580; pin guard notes ~1900-1925) | local file | read | +| L3 | `F:\Projects\Eve` and `F:\Projects\Brokkr` csprojs and git file lists | local files | read | +| L4 | voidbot `search_history` (two queries), `search_sources` (one query) | MCP | queried | + +Anything not in a "fetched" row above is not claimed from the web. + +## 1. Established practice: who pays for a breaking upstream change, and when + +- **Abseil / live at head (S1, S2).** Recommends users track the latest code. Says it will provide a tool to perform an API refactor for well-behaved code (S1). Requires of downstream: show how you call it, stay within the compatibility guidelines; otherwise fork or use periodic supported branches (about every 6 months, patched 2 years) (S1). Says every change is potentially breaking for someone; breaks are announced in commit messages and LTS notes, and users are asked to report undetected breaks on GitHub (S2). Users should be able to temporarily pin to an earlier commit if a newer change is a problem (S2). S1/S2 do not mention Rosie, the Beyonce rule, or caller-fixing; the fetched pages describe tooling and a feedback channel only. +- **Live at Head as described in SWE at Google ch. 21 (S3).** Providers are expected to test changes against downstream dependents before committing; breaking changes wait until dependents are updated or an automated migration tool exists. Consumers must keep meaningful tests. Stated preconditions: unit tests across dependencies run in CI, a visible reverse-dependency graph, substantial compute, no pinning of dependencies, provider acceptance of the downstream-testing duty. Called "costly but effective" inside Google and "largely unproven at scale." Notes that under SemVer a major bump costs maintainers little and many indirect users a lot. Ch. 16 (S4) does not address live at head; it states large-scale changes across trunk are costly and tooling-supported. +- **Go MVS (S5).** Selects the oldest version satisfying all declared minimums. Publishing a new version changes nobody's build; a consumer gets it only when a requirement names it, directly or through another module. Upgrades are explicit edits to `go.mod` by the top-level module. Compatibility rule: newer versions inside one major version are expected to work where older did; a major version is a different module identity. Cost of a breaking change falls on whoever opts into the new version, at the time they opt in. +- **Cargo (S6).** Git dependency without `rev` takes the default-branch tip at first build and fixes that commit in `Cargo.lock`; changes arrive only on `cargo update`. A `rev` in `Cargo.toml` pins explicitly; the Cargo book calls hand-managed SHAs "tedious and error prone" and prefers the lockfile. Consumer pays at `cargo update` time. +- **npm (S7).** `package-lock.json` records the exact tree; installs stay on it until the lockfile changes and is committed. It is not published with a package. The page gives no timeline for when updates reach a consumer. +- **Chromium DEPS (S8).** `DEPS` stores dependency URLs and commit hashes (often in `vars`). Developers roll with `gclient setdep`, `roll-dep`, or manual `git update-index`; some projects (Skia) are autorolled. The page says manual rolling after an upstream change is good practice to confirm the roll compiles and tests pass. It does not describe revert procedure. Cost is paid by the roller at roll time. +- **Git submodules (S9).** The superproject records one commit SHA (gitlink, mode 160000). Upstream commits never move the pin; it moves when someone updates the checkout and commits the new SHA in the superproject. Collaborators receive the pin via pull and must run `git submodule update`. + +## 2. Internal precedent + +- **Aetheria pin (L1).** `Directory.Build.props` holds `CultLibRevision` (currently `45c2f4006f5f7dab357426cabd3197e8d0a17f66`) and `CultMathRevision` (`9c1b06cd703bfa44352050e819b0d0659cf5978a`). `Directory.Build.targets` target `VerifyCultLibRevision` (BeforeTargets `ResolveProjectReferences`) errors if the sibling checkout's `git rev-parse HEAD` differs from the pin, or if `git status --porcelain` is non-empty; escape hatch `SkipCultLibRevisionCheck=true`. File is 39 lines; the pin is one pair of properties per pinned package. +- **Why added (L1 git).** Commit `4f4beb0f` (2026-09-14, "Pin CultLib for headless builds"): "Directory.Build.targets carries Delvehold's VerifyCultLibRevision guard, checking for CultMath.csproj: HEAD must equal the pin and the worktree must be clean." Commit `e91ba369` (2026-09-24, "Pin CultLib per package, as Unity's manifest does"): headless build must compile the revisions Unity consumes; a single pin would have compiled six commits of newer CultCache under headless tests while Unity stays on its tag. The Aetheria cut doc (L2, ~line 1905) calls the guard "the only working pin for a sibling checkout." Aetheria's `Packages/manifest.json` pins Unity packages by git tag. +- **Pin guard already in Delvehold (L4/search_sources hit, `Delvehold:Directory.Build.targets`)** and a variant in AetheriaEve (archived, `Aetheria.State.Dependencies.targets`, also checks a Unity package tree hash). Delvehold's guard checks `GameCult.Mesh.csproj` exists, HEAD equals `CultLibRevision`, tree clean. Date of Delvehold's first guard: not determined. +- **Consumer table (L2, Q6 in Aetheria `docs/cultcache-migration-cut.md`).** Aquarium, Ymir, AquaSynth, Mimir, Gjallar: ".NET sibling, unpinned", compile against main at once. Brokkr, Eve, EveUnity, EvePlugins: "Unity package / sibling", pin "various". Delvehold: sibling with revision guard at `334e60f`, 52 behind, "Not until it re-pins." Heimdall: submodule `b6b1d9c`. Rust consumers (Idunn, Epiphany, Odin, Ghostlight, Muninn, Ratatoskr, CodexConnector): Cargo by commit. +- **Operator discussion (L4).** voidbot `search_history` ("pin CultLib revision vs track main for consumers"; "Brokkr Eve sibling CultLib checkout build pin") returned no relevant operator statement on pinning versus tracking; hits were unrelated Discord chatter. Nearest operator words found in the Discord index: Metacrat, 2026-05-09, "Oh, anyone using CultLib will still be paying the blood price, you can count on that" (message 1502671320406687894; context not fetched, so its subject is not confirmed). No operator quote on pinning versus tracking was found in memory files grepped (`C:\Users\Meta\.claude\projects\F--Projects\memory`, pattern pin/track main/unpinned/sibling) beyond a retired Idunn pin line and Odin/Muninn pin-branch coordination notes. +- Aetheria cut doc operator mentions near pins are about content (Schematic-shape drawer, 2026-09-14; Studio grouping, 2026-09-17), not pinning policy. + +## 3. Current state of the two consumers (L3) + +- **Eve.** `packages/org.gamecult.eve.surface/GameCult.Eve.Surface.csproj:16` `CultLibRoot` defaults to `$(MSBuildThisFileDirectory)..\..\..\CultLib`; lines 20-21 ProjectReference `$(CultLibRoot)\src\GameCult.Caching\...` and `...\GameCult.Mesh\...`. `tests/GameCult.Eve.Surface.Serialization.Tests/*.csproj:19` references `$(CultLibRoot)\src\GameCult.Mesh\...` (and Eve.Surface by relative path). `git ls-files` shows only `packages/org.gamecult.eve.surface/Directory.Build.props` (sets only BaseOutputPath/BaseIntermediateOutputPath/MSBuildProjectExtensionsPath, no revision). No `Directory.Build.targets`, no `global.json`, no `.gitmodules`, no revision property or check. Eve also holds Cargo.toml/Cargo.lock (Rust side, not examined). +- **Brokkr.** `brokkr-command/Brokkr.Command.csproj:7`, `brokkr-contracts.Tests/Brokkr.Contracts.Tests.csproj:7`, `brokkr-contracts/Brokkr.Contracts.csproj:8`: `CultLibRoot` defaults to `..\..\CultLib`. ProjectReferences: Caching, Caching.MessagePack, Mesh, Networking (by project). No `Directory.Build.*`, no `.gitmodules` in `git ls-files`; no revision property or check. +- Neither repo contains `CultLibRevision`, a `VerifyCultLibRevision` target, or any HEAD check. Both build whatever `F:\Projects\CultLib` currently holds (including dirty or other-branch states). +- Fork text claims (not re-verified here): Eve.Surface has 0 `[CultElementId]`; Brokkr `[CultDocument]` files show no object lists by grep. + +## Gaps + +- S10 (Chromium infra) 404; Chromium autoroller behavior beyond S8 not fetched. +- Rosie / Beyonce rule not found in fetched pages; not cited from memory. +- No operator statement on pin-versus-track found in voidbot or the memory grep. diff --git a/docs/runtime-parity-scope.md b/docs/runtime-parity-scope.md index 2c6f2211..e001bb1d 100644 --- a/docs/runtime-parity-scope.md +++ b/docs/runtime-parity-scope.md @@ -15,6 +15,9 @@ Cross-runtime parity has three layers: without translation folklore. Every store reader refuses, by name, a header or record slot it does not understand (vectors: `tests/vectors/document-variants-c0`). + Element ids: a store that can hold them declares `cultcache.store.v3`; TypeScript, + Python and Rust read it and keep the header, and readers older than ids refuse it + (vectors: `tests/vectors/document-variants-c2a`). Document variants: C# resolves them (`cultcache.store.v2`); every other runtime, CultMesh single-file reads, the directory store and CultNet refuse a variant by name (vector written by C#: `tests/vectors/document-variants-c1`). @@ -41,8 +44,8 @@ to speak CultNet or participate in CultMesh. | C# | Reference implementation and production game/server substrate | CultCache typed documents, CultNet schema-v0, LiteNetLib production path with adapter profile, shared transport profiles, shared RUDP socket transport, C# multi-peer RUDP listener and schema service host, C# interop RUDP serve/dial path, connected RUDP CultMesh client helpers, RUDP-backed schema-client selection for `rudp://` service endpoints, RUDP reconnect policy/controller and reconnect-loop helper, LiteNetLib client reconnect-controller adoption, schema-client transport port for service request/response clients, CultMesh node/session/catalog/stream/simulation/RUDP surfaces, authorized peer lookup | No need to abandon LiteNetLib before the CultNet adapter fully owns production service adoption | | Python | Package runtime, local daemon endpoint, raw interop peer, typed state participant | CultCache/CultNet/CultMesh package surface, branded CultMesh node/schema/shard/Verse/peer/authority/stream/RUDP ergonomics, authorized peer lookup, shard catalogs/logs, schema-transport factory for raw client request/subscription flows, local TCP and RUDP CultMesh server listeners, Python interop RUDP serve/dial path, session/prediction/simulation fact helpers, RUDP codec/session/socket transport, RUDP reconnect policy/controller and reconnect-loop helper, schema-v0 MessagePack over RUDP evidence, benchmark evidence against C# cache paths | Universal C# throughput, full C# daemon/server stack, and RUDP as the only public service pipe | | TypeScript | Node/browser-adjacent projection and interop runtime | CultCache parity tests, CultNet schema-v0 peer, advertised schema/shard catalog wire contracts plus request/sync helpers, transport profiles, TCP-framed transport port and peer helper, shared RUDP socket transport, RUDP interop serve/dial path, RUDP reconnect policy/controller, RUDP reconnect-loop helper, branded CultMesh node/schema/shard/Verse/peer/authority/stream/RUDP ergonomics, authorized peer lookup | Full C# game session/server stack | -| Rust | Low-level cache/transport/runtime substrate | CultCache/CultNet interop lane, schema and shard catalog helpers, typed same-schema alias hydration from raw snapshots, transport profiles, TCP-framed port, shared RUDP codec/session/socket transport, Rust interop RUDP serve/dial path, connected RUDP CultMesh client helpers, RUDP reconnect policy/controller and reconnect-loop helper, schema-v0 MessagePack over RUDP evidence, small CultMesh facade for schema registry, shard catalog, peer cards, authority leases, authorized peer lookup, RUDP endpoint parsing, and peer-card/authorized-peer RUDP client construction | Full CultMesh game-session/server facade or production game-session server. `packages/cultmesh-rs` now holds a Rust stream and session surface -- node, document sets, RUDP document publication, stream descriptors, negotiation, stream catalog, and shared-memory frame rings -- but no interop lane exercises it, so it carries no cross-runtime parity claim | -| Kotlin | JVM/Android client substrate | Typed MessagePack cache, one-call typed snapshot document sync through local same-schema aliases, built-in schema catalog advertisement, schema/shard catalogs with shared schema-message transport fetch/sync helpers, authority leases, authorized peer lookup, stream catalog, branded `CultMesh` facade, WebSocket transport profile/connection helper, RUDP client/server sugar, connected RUDP CultMesh client helpers, Kotlin interop RUDP serve/dial path, RUDP reconnect policy/controller and reconnect-loop helper, schema-v0 MessagePack over RUDP evidence | C# SoA store (parked at `parked/cultcache-soa`), full C# server daemon, or Unity hot-loop memory layout | +| Rust | Low-level cache/transport/runtime substrate | CultCache/CultNet interop lane, schema and shard catalog helpers, typed hydration of raw snapshots into the receiver's own registered type for each schema id, transport profiles, TCP-framed port, shared RUDP codec/session/socket transport, Rust interop RUDP serve/dial path, connected RUDP CultMesh client helpers, RUDP reconnect policy/controller and reconnect-loop helper, schema-v0 MessagePack over RUDP evidence, small CultMesh facade for schema registry, shard catalog, peer cards, authority leases, authorized peer lookup, RUDP endpoint parsing, and peer-card/authorized-peer RUDP client construction | Full CultMesh game-session/server facade or production game-session server. `packages/cultmesh-rs` now holds a Rust stream and session surface -- node, document sets, RUDP document publication, stream descriptors, negotiation, stream catalog, and shared-memory frame rings -- but no interop lane exercises it, so it carries no cross-runtime parity claim | +| Kotlin | JVM/Android client substrate | Typed MessagePack cache, one-call typed snapshot document sync into the receiver's own registered type for the schema, built-in schema catalog advertisement, schema/shard catalogs with shared schema-message transport fetch/sync helpers, authority leases, authorized peer lookup, stream catalog, branded `CultMesh` facade, WebSocket transport profile/connection helper, RUDP client/server sugar, connected RUDP CultMesh client helpers, Kotlin interop RUDP serve/dial path, RUDP reconnect policy/controller and reconnect-loop helper, schema-v0 MessagePack over RUDP evidence | C# SoA store (parked at `parked/cultcache-soa`), full C# server daemon, or Unity hot-loop memory layout | ## Evidence Gates diff --git a/package.json b/package.json index c9b79e47..a79661f2 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,7 @@ "scripts": { "build:ts": "node scripts/run-typescript-workspaces.mjs build", "test:ts": "node scripts/run-typescript-workspaces.mjs test", - "test:pack:ts": "node scripts/test-typescript-package-closure.mjs", + "test:pack:ts": "node scripts/test-typescript-package-closure.mjs && node scripts/test-declared-dependency-ranges.mjs", "test:eve-browser-network": "node scripts/verify-eve-browser-network.mjs" }, "devDependencies": { diff --git a/packages/cultcache-py/CHANGELOG.md b/packages/cultcache-py/CHANGELOG.md new file mode 100644 index 00000000..d2f76628 --- /dev/null +++ b/packages/cultcache-py/CHANGELOG.md @@ -0,0 +1,15 @@ +# Changelog + +## Unreleased + +### Breaking + +- A single-file store written as `cultcache.store.v3` is read and written: a record that holds an element id (an owned list element with an `[CultElementId]` member, carried through every write) marks the store v3, and a v3 store is carried through a rewrite with its ids. A runtime from before this change refuses a v3 store by name. +- A write over a file the reader refuses (`StoreUnreadableError`, from `push`, `push_all` and `delete`) refuses and leaves the file as it is. A flush used to replace such a file. A write set cannot merge with a disk it cannot read, so the file is repaired or removed by its owner before the store writes again. +- Added `StoreUnreadableError` (a `ValueError`) for a store file this runtime cannot read, and `SchemaConflictError` (a `ValueError`) for a + write the catalog cannot describe. A store is exactly three slots; a schema is never recovered from a record's payload. +- The single-file writer derives the catalog from the records it writes: one entry per carried schema id, the registered entry over an + arrived one, written as chosen and never merged. A cache stamps a record with its registered schema's id. +- Every registration refusal is now a `SchemaConflictError`, a `ValueError`, where it was a `CultCacheError`, a `RuntimeError`: registering a document type or a schema name twice no longer raises what `except CultCacheError` or `except RuntimeError` catches. +- Two documents that own one schema id are refused at registration (`SchemaConflictError`). 2.0.1 accepted them. A document may still list another's id as compatible. +- A loaded record resolves by its schema id first: the document that owns the id, else the one document that lists it as compatible; several listers and no owner refuse the record with `SchemaConflictError`. The record's type name resolves it only when its schema id names no registered document. 2.0.1 resolved by the record's type name alone. diff --git a/packages/cultcache-py/src/cultcache_py/__init__.py b/packages/cultcache-py/src/cultcache_py/__init__.py index dbfac403..aabd21bf 100644 --- a/packages/cultcache-py/src/cultcache_py/__init__.py +++ b/packages/cultcache-py/src/cultcache_py/__init__.py @@ -17,7 +17,7 @@ from .documents import ( define_union, ) from .leaves import SlotCodec -from .stores import JsonLinesBackingStore, SingleFileMessagePackBackingStore +from .stores import JsonLinesBackingStore, SchemaConflictError, SingleFileMessagePackBackingStore, StoreUnreadableError __all__ = [ "BackingStore", @@ -32,8 +32,10 @@ __all__ = [ "JsonLinesBackingStore", "MAX_PAYLOAD_DEPTH", "MAX_SLOT_KEY", + "SchemaConflictError", "SingleFileMessagePackBackingStore", "SlotCodec", + "StoreUnreadableError", "UnionArm", "leaves", "database_entry_field", diff --git a/packages/cultcache-py/src/cultcache_py/cache.py b/packages/cultcache-py/src/cultcache_py/cache.py index 0da2c04a..8f4ac759 100644 --- a/packages/cultcache-py/src/cultcache_py/cache.py +++ b/packages/cultcache-py/src/cultcache_py/cache.py @@ -7,6 +7,7 @@ from typing import Any, Callable, Generic, TypeVar from .backing_store import BackingStore, CultCacheEnvelope from .documents import DocumentDefinition, extract_value +from .stores import SchemaConflictError T = TypeVar("T") F = TypeVar("F", bound=Callable[..., Any]) @@ -138,11 +139,28 @@ class CultCache: self._register_document_type(document) def _register_document_type(self, document: DocumentDefinition[Any]) -> None: + # Every registration refusal is a SchemaConflictError naming the new document's schema id and both schema names. + entry = document.catalog_entry() + schema_name = entry.schema_name + + def refuse(registered: DocumentDefinition[Any], what: str) -> SchemaConflictError: + return SchemaConflictError( + entry.schema_id, + [registered.catalog_entry().schema_name, schema_name], + "", + message=f"{what} is already registered to document type {registered.type!r} and cannot also be claimed by " + f"{document.type!r}.", + ) + if document.type in self._state.documents: - raise CultCacheError(f"Document type already registered: {document.type}") - schema_name = document.catalog_entry().schema_name + raise refuse(self._state.documents[document.type], f"Document type {document.type!r}") if schema_name in self._state.documents_by_schema_name: - raise CultCacheError(f"Document schema name already registered: {schema_name}") + raise refuse(self._state.documents_by_schema_name[schema_name], f"Schema name {schema_name!r}") + # One document owns each schema id. Others may list it as compatible beside its owner: a record under the id resolves to + # its owner, and to a lister only when nothing owns it. Neither depends on the order documents were registered in. + for registered in self._state.documents.values(): + if registered.catalog_entry().schema_id == entry.schema_id: + raise refuse(registered, f"Schema id {entry.schema_id!r}") name_extractors = dict(self._state.name_extractors) index_extractors = {type: dict(indexes) for type, indexes in self._state.index_extractors.items()} if document.name is not None: @@ -280,6 +298,7 @@ class CultCache: if envelope.key != GLOBAL_KEY: loaded_legacy_global_keys[envelope.type] = envelope.key envelope = replace(envelope, key=GLOBAL_KEY) + envelope = self._stamped(document, envelope) value = document.decode_payload(envelope.payload) loaded_values.setdefault(envelope.type, {})[envelope.key] = value loaded_envelopes.setdefault(envelope.type, {})[envelope.key] = envelope @@ -372,6 +391,7 @@ class CultCache: def put_envelope(self, document: DocumentDefinition[T], envelope: CultCacheEnvelope) -> T: self._assert_registered(document) self._check_envelope(document, envelope) + envelope = self._stamped(document, envelope) value = document.decode_payload(envelope.payload) self._write(document, [(envelope, value)], batch=False) return value @@ -380,12 +400,21 @@ class CultCache: def put_envelopes(self, document: DocumentDefinition[T], envelopes: list[CultCacheEnvelope]) -> list[T]: self._assert_registered(document) values: list[T] = [] + stamped: list[CultCacheEnvelope] = [] for envelope in envelopes: self._check_envelope(document, envelope) + stamped.append(self._stamped(document, envelope)) values.append(document.decode_payload(envelope.payload)) - self._write(document, list(zip(envelopes, values)), batch=True) + self._write(document, list(zip(stamped, values)), batch=True) return values + @staticmethod + def _stamped(document: DocumentDefinition[Any], envelope: CultCacheEnvelope) -> CultCacheEnvelope: + """The record under its registered schema's id, whatever id the envelope arrived under, so the catalog entry a store + writes is the one its schema id names.""" + entry = document.catalog_entry() + return replace(envelope, schema_id=entry.schema_id, catalog_entry=entry) + @staticmethod def _check_envelope(document: DocumentDefinition[Any], envelope: CultCacheEnvelope) -> None: if envelope.type != document.type: @@ -581,6 +610,27 @@ class CultCache: raise CultCacheError(f"Document type is not global: {document.type}") def _resolve_document_for_envelope(self, envelope: CultCacheEnvelope) -> DocumentDefinition[Any] | None: + """A record resolves by its schema id: the document that owns the id, else the one document that lists it as compatible. + Several listers and no owner name no single document, and the record is refused. The name its catalog carries is + metadata; it names a document only when the id names none (a store from a runtime whose schema ids this one cannot + know).""" + if envelope.schema_id: + entries = [(document, document.catalog_entry()) for document in self._state.documents.values()] + for document, entry in entries: + if entry.schema_id == envelope.schema_id: + return document + listers = [document for document, entry in entries if envelope.schema_id in entry.compatible_schema_ids] + if len(listers) > 1: + raise SchemaConflictError( + envelope.schema_id, + sorted(document.catalog_entry().schema_name for document in listers), + envelope.key, + message=f"Schema id {envelope.schema_id!r} of record {envelope.key!r} is owned by no registered document and " + f"declared compatible by several ({', '.join(sorted(document.type for document in listers))}), so it resolves " + "to none. Register the document that owns the id, or declare it on one document.", + ) + if listers: + return listers[0] document = self._state.documents.get(envelope.type) if document is not None: return document diff --git a/packages/cultcache-py/src/cultcache_py/stores.py b/packages/cultcache-py/src/cultcache_py/stores.py index 704a3851..c9c76b06 100644 --- a/packages/cultcache-py/src/cultcache_py/stores.py +++ b/packages/cultcache-py/src/cultcache_py/stores.py @@ -14,11 +14,42 @@ from .backing_store import ( ) STORE_FORMAT_VERSION = "cultcache.store.v1" +# A store that can hold element ids. A reader older than element ids refuses this header: it would skip the id slots of a +# nested element and rewrite the element without them. This runtime writes it back only for a store it read under it; it +# does not decide when a document carries ids. +STORE_FORMAT_ELEMENT_IDS = "cultcache.store.v3" +_READABLE_STORE_FORMATS = (STORE_FORMAT_VERSION, STORE_FORMAT_ELEMENT_IDS) _STORE_FORMAT_PREFIX = "cultcache.store." _PERSISTED_RECORD_SLOTS = 4 _Contents = TypeVar("_Contents") +class StoreUnreadableError(ValueError): + """A store file this runtime cannot read: not exactly one complete store (truncated, bytes after it, a missing or extra + slot), a header or record it does not know, or a body it cannot decode. Open, push, delete and push_all refuse a file + with this error, and a refused file is left as it was. The cause is `__cause__`.""" + + def __init__(self, path: Path, cause: BaseException) -> None: + super().__init__(f"CultCache store {path} is not readable: {cause}") + self.path = path + + +class SchemaConflictError(ValueError): + """A write the catalog cannot describe: records of different types under one schema id, two arrived schemas that share an id + and disagree on the schema name, or a record no chosen entry publishes. Nothing is written. It names the id, the names + involved (`schema_names`) and a record key (`record_key`). A cache raises it too, with an empty record key, when a + document registers a schema id another registered document already carries.""" + + def __init__(self, schema_id: str, schema_names: list[str], record_key: str, *, message: str | None = None) -> None: + super().__init__( + message + or f"Schema id {schema_id!r} cannot describe record {record_key!r}: {', '.join(repr(name) for name in schema_names)}" + ) + self.schema_id = schema_id + self.schema_names = schema_names + self.record_key = record_key + + class JsonLinesBackingStore: def __init__(self, path: str | os.PathLike[str]) -> None: self.path = Path(path) @@ -86,43 +117,45 @@ class SingleFileMessagePackBackingStore: def __init__(self, path: str | os.PathLike[str]) -> None: self.path = Path(path) self._lock = threading.RLock() + # The header this store read; a rewrite keeps it, so a store that holds element ids never sheds its marker. + self._format = STORE_FORMAT_VERSION def pull_all(self) -> list[CultCacheEnvelope]: msgpack = self._msgpack() with self._lock: # An empty file is not a store (no writer leaves one), so it is decoded and refused. + # The disk decides the header: a file that is gone is not marked. + self._format = STORE_FORMAT_VERSION data = _read_store(self.path, Path.read_bytes) if data is None: return [] - decoded = msgpack.unpackb(data, raw=False) - snapshot = _decode_v1_snapshot(decoded) - if snapshot is not None: - return snapshot - return _decode_legacy_envelopes(decoded, msgpack) + try: + decoded = msgpack.unpackb(data, raw=False) + snapshot = _decode_snapshot(decoded) + if snapshot is not None: + self._format, envelopes = snapshot + return envelopes + return _decode_legacy_envelopes(decoded, msgpack) + except Exception as exc: + raise StoreUnreadableError(self.path, exc) from exc def push(self, envelope: CultCacheEnvelope) -> None: with self._lock: - existing = {(item.type, item.key): item for item in self.pull_all()} - existing[(envelope.type, envelope.key)] = envelope - self._replace_all(list(existing.values())) + self._replace_all([envelope], self.pull_all()) def push_all(self, envelopes: list[CultCacheEnvelope]) -> None: with self._lock: - existing = {(item.type, item.key): item for item in self.pull_all()} - for envelope in envelopes: - existing[(envelope.type, envelope.key)] = envelope - self._replace_all(list(existing.values())) + self._replace_all(list(envelopes), self.pull_all()) def delete(self, type: str, key: str) -> None: with self._lock: - existing = [item for item in self.pull_all() if not (item.type == type and item.key == key)] - self._replace_all(existing) + self._replace_all([], [item for item in self.pull_all() if not (item.type == type and item.key == key)]) - def _replace_all(self, envelopes: list[CultCacheEnvelope]) -> None: + def _replace_all(self, supplied: list[CultCacheEnvelope], arrived: list[CultCacheEnvelope]) -> None: msgpack = self._msgpack() self.path.parent.mkdir(parents=True, exist_ok=True) temp = self.path.with_suffix(self.path.suffix + ".tmp") - temp.write_bytes(msgpack.packb(_encode_v1_snapshot(envelopes), use_bin_type=True)) + temp.write_bytes(msgpack.packb(_encode_snapshot(supplied, arrived, self._format), use_bin_type=True)) temp.replace(self.path) @staticmethod @@ -137,22 +170,96 @@ class SingleFileMessagePackBackingStore: return msgpack -def _encode_v1_snapshot(envelopes: list[CultCacheEnvelope]) -> list[Any]: - catalog_by_schema_id: dict[str, CultCacheSchemaCatalogEntry] = {} - for envelope in envelopes: - schema_id = _schema_id_for(envelope) - if schema_id not in catalog_by_schema_id: - catalog_by_schema_id[schema_id] = envelope.catalog_entry or _default_catalog_entry(envelope) +def canonical(candidates: Any) -> list[CultCacheSchemaCatalogEntry]: + """Entries that tie, in one fixed order, so the catalog does not depend on the order they arrive in.""" + return sorted(candidates, key=lambda entry: (entry.schema_name, entry.content_hash, ",".join(entry.compatible_schema_ids))) + +def _derive_catalog(records: list[tuple[CultCacheEnvelope, bool]]) -> list[CultCacheSchemaCatalogEntry]: + """The catalog a write leaves, derived from the records being written and nothing else. For each schema id a record carries, + one entry that publishes it: the entry that owns the id (a registered one, else one that arrived with the records), else one + that lists it as compatible (registered first). Entries are written as chosen, never merged; a registered entry wins over an + arrived one with the same own id. Records of different types under one id, two arrived entries with the same own id and + different schema names, or a record no chosen entry publishes, refuse the write. `records` are (envelope, registered): an + envelope the caller supplied is registered, one read back from the file is arrived.""" + entries: list[tuple[CultCacheSchemaCatalogEntry, bool]] = [] + by_id: dict[str, list[CultCacheEnvelope]] = {} + tiers: dict[str, list[tuple[CultCacheEnvelope, bool, CultCacheSchemaCatalogEntry]]] = {} + for envelope, registered in records: + schema_id = _schema_id_for(envelope) + supplied = envelope.catalog_entry + publishes = supplied is not None and (supplied.schema_id == schema_id or schema_id in supplied.compatible_schema_ids) + entry = supplied if publishes else _default_catalog_entry(envelope) + entries.append((entry, registered)) + by_id.setdefault(schema_id, []).append(envelope) + tiers.setdefault(schema_id, []).append((envelope, registered, entry)) + + chosen: dict[str, tuple[CultCacheSchemaCatalogEntry, bool]] = {} + for schema_id in sorted(by_id): + carrying = by_id[schema_id] + record_key = min(envelope.key for envelope in carrying) + # Records of one tier under one id are one type. Across tiers the id keeps the type its records resolve to, so a write may + # not retype records that are already there: unless the entries of both tiers own the id, which is a rename of the schema + # under a stable id, and the registered descriptor wins. A read-back record's type is its schema's name, so tiers compare + # the names of the entries that publish the id. + for tier in (True, False): + types = sorted({envelope.type for envelope, registered, _ in tiers[schema_id] if registered == tier}) + if len(types) > 1: + raise SchemaConflictError(schema_id, types, record_key) + registered_names = sorted({entry.schema_name for _, registered, entry in tiers[schema_id] if registered}) + arrived_names = sorted({entry.schema_name for _, registered, entry in tiers[schema_id] if not registered}) + owned_by = lambda tier: any(entry.schema_id == schema_id for _, registered, entry in tiers[schema_id] if registered == tier) + if registered_names and arrived_names and registered_names[0] != arrived_names[0] and not (owned_by(True) and owned_by(False)): + raise SchemaConflictError(schema_id, sorted([registered_names[0], arrived_names[0]]), record_key) + own_registered = next(iter(canonical(entry for entry, registered in entries if registered and entry.schema_id == schema_id)), None) + own_arrived = canonical(entry for entry, registered in entries if not registered and entry.schema_id == schema_id) + own_registered_all = canonical(entry for entry, registered in entries if registered and entry.schema_id == schema_id) + # Entries of one tier that share an own id must be one schema; a registered entry settles a disagreement among arrived ones. + names = sorted({entry.schema_name for entry in (own_registered_all or own_arrived)}) + if len(names) > 1: + raise SchemaConflictError(schema_id, names, record_key) + if own_registered is not None: + pick, registered_pick = own_registered, True + elif own_arrived: + pick, registered_pick = own_arrived[0], False + else: + listed_registered = next(iter(canonical(entry for entry, registered in entries if registered and schema_id in entry.compatible_schema_ids)), None) + listed_arrived = next(iter(canonical(entry for entry, registered in entries if not registered and schema_id in entry.compatible_schema_ids)), None) + pick = listed_registered if listed_registered is not None else listed_arrived + registered_pick = listed_registered is not None + if pick is None: + continue + prior = chosen.get(pick.schema_id) + if prior is not None and prior[1] and not registered_pick: + continue + chosen[pick.schema_id] = (pick, registered_pick) + + for schema_id in sorted(by_id): + if not any(entry.schema_id == schema_id or schema_id in entry.compatible_schema_ids for entry, _ in chosen.values()): + raise SchemaConflictError( + schema_id, sorted(entry.schema_name for entry, _ in chosen.values()), min(envelope.key for envelope in by_id[schema_id]) + ) + return [entry for entry, _ in chosen.values()] + + +def _encode_snapshot( + supplied: list[CultCacheEnvelope], arrived: list[CultCacheEnvelope], format_version: str +) -> list[Any]: + replaced = {(envelope.type, envelope.key) for envelope in supplied} + kept = [envelope for envelope in arrived if (envelope.type, envelope.key) not in replaced] + envelopes = [*kept, *supplied] catalog = [ _encode_catalog_entry(entry) - for entry in sorted(catalog_by_schema_id.values(), key=lambda item: item.schema_name) + for entry in sorted( + _derive_catalog([*((envelope, False) for envelope in kept), *((envelope, True) for envelope in supplied)]), + key=lambda item: (item.schema_name, item.schema_id), + ) ] records = [ [envelope.key, _schema_id_for(envelope), envelope.stored_at, envelope.payload] for envelope in sorted(envelopes, key=lambda item: item.key) ] - return [STORE_FORMAT_VERSION, catalog, records] + return [format_version, catalog, records] def _read_store(path: Path, read: Callable[[Path], _Contents]) -> _Contents | None: @@ -186,25 +293,29 @@ def _describe_header(header: str) -> str: return f"an unrecognised {_STORE_FORMAT_PREFIX}* header of {len(header.encode('utf-8'))} bytes" -def _decode_v1_snapshot(decoded: Any) -> list[CultCacheEnvelope] | None: +def _decode_snapshot(decoded: Any) -> tuple[str, list[CultCacheEnvelope]] | None: if not isinstance(decoded, list) or not decoded: return None - if isinstance(decoded[0], str) and decoded[0].startswith(_STORE_FORMAT_PREFIX) and decoded[0] != STORE_FORMAT_VERSION: + if isinstance(decoded[0], str) and decoded[0].startswith(_STORE_FORMAT_PREFIX) and decoded[0] not in _READABLE_STORE_FORMATS: raise ValueError( f"CultCache store format {_describe_header(decoded[0])} is not one this runtime reads; " - f"it reads {STORE_FORMAT_VERSION!r} only" + f"it reads {STORE_FORMAT_VERSION!r} and {STORE_FORMAT_ELEMENT_IDS!r} only" ) - if decoded[0] != STORE_FORMAT_VERSION: + if decoded[0] not in _READABLE_STORE_FORMATS: return None - if len(decoded) < 3 or not isinstance(decoded[1], list) or not isinstance(decoded[2], list): + if len(decoded) != 3: + raise ValueError(f"CultCache store has {len(decoded)} top-level slots; this runtime reads 3 (header, schema catalog, records)") + if not isinstance(decoded[1], list) or not isinstance(decoded[2], list): raise ValueError("CultCache v1 snapshot must contain a schema catalog and record array") + # An id names the entry that owns it; only an id no entry owns names the entry that lists it as compatible. + entries = [_decode_catalog_entry(raw_entry) for raw_entry in decoded[1]] catalog_by_schema_id: dict[str, CultCacheSchemaCatalogEntry] = {} - for raw_entry in decoded[1]: - entry = _decode_catalog_entry(raw_entry) - catalog_by_schema_id[entry.schema_id] = entry + for entry in entries: + catalog_by_schema_id.setdefault(entry.schema_id, entry) + for entry in entries: for compatible_schema_id in entry.compatible_schema_ids: - catalog_by_schema_id[compatible_schema_id] = entry + catalog_by_schema_id.setdefault(compatible_schema_id, entry) envelopes: list[CultCacheEnvelope] = [] for raw_record in decoded[2]: @@ -225,19 +336,7 @@ def _decode_v1_snapshot(decoded: Any) -> list[CultCacheEnvelope] | None: raise ValueError("CultCache persisted records must declare storedAt") catalog_entry = catalog_by_schema_id.get(schema_id) if catalog_entry is None: - schema_version = _infer_schema_version_from_payload(_normalize_payload(payload)) - schema_name = _infer_schema_name(schema_version) if schema_version is not None else None - if schema_name is None: - raise ValueError(f'CultCache persisted record "{key}" references missing schema id "{schema_id}"') - catalog_entry = CultCacheSchemaCatalogEntry( - schema_id=schema_id, - schema_name=schema_name, - schema_version=schema_version, - content_hash=schema_id, - canonical_schema_json="", - compatible_schema_ids=(), - members=(), - ) + raise ValueError(f"CultCache persisted record '{key}' references schema id '{schema_id}', which the store's catalog does not publish") normalized_payload = _normalize_payload(payload) envelopes.append( CultCacheEnvelope( @@ -249,7 +348,7 @@ def _decode_v1_snapshot(decoded: Any) -> list[CultCacheEnvelope] | None: catalog_entry=catalog_entry, ) ) - return envelopes + return decoded[0], envelopes def _decode_legacy_envelopes(decoded: Any, msgpack: Any) -> list[CultCacheEnvelope]: @@ -376,27 +475,3 @@ def _normalize_payload(payload: Any) -> bytes: if isinstance(payload, list) and all(isinstance(item, int) and 0 <= item <= 255 for item in payload): return bytes(payload) raise ValueError("CultCache record payload must be binary MessagePack bytes") - - -def _infer_schema_version_from_payload(payload: bytes) -> str | None: - try: - msgpack = SingleFileMessagePackBackingStore._msgpack() - decoded = msgpack.unpackb(payload, raw=False) - except Exception: - return None - if isinstance(decoded, list) and decoded and isinstance(decoded[0], str): - return decoded[0] - if isinstance(decoded, dict): - value = decoded.get("schemaVersion", decoded.get("schema_version")) - return value if isinstance(value, str) else None - return None - - -def _infer_schema_name(schema_version: str) -> str | None: - marker = schema_version.rfind(".v") - if marker <= 0: - return None - version = schema_version[marker + 2:] - if not version or not version.isdigit(): - return None - return schema_version[:marker] diff --git a/packages/cultcache-py/tests/test_cultcache.py b/packages/cultcache-py/tests/test_cultcache.py index 6e85f833..f5bad365 100644 --- a/packages/cultcache-py/tests/test_cultcache.py +++ b/packages/cultcache-py/tests/test_cultcache.py @@ -12,6 +12,7 @@ from cultcache_py import ( CultCache, JsonLinesBackingStore, SingleFileMessagePackBackingStore, + StoreUnreadableError, define_database_entry_type, define_document_type, ) @@ -186,7 +187,7 @@ class CultCacheTests(unittest.TestCase): self.assertEqual(raw[2][0][0], "app") self.assertEqual(raw[2][0][1], "settings") - def test_messagepack_store_recovers_schema_stamped_record_missing_catalog_entry(self) -> None: + def test_messagepack_store_refuses_a_record_whose_schema_the_catalog_does_not_publish(self) -> None: try: import msgpack # type: ignore except ModuleNotFoundError: @@ -227,11 +228,490 @@ class CultCacheTests(unittest.TestCase): .build() ) - cache.pull_all_backing_stores() - self.assertEqual(cache.get_required(document, "record-1")["value"], "still readable") - envelope = cache.get_required_envelope(document, "record-1") - self.assertEqual(envelope.type, "runtime-policy") - self.assertEqual(envelope.schema_id, "sha256:stale-schema-id-from-cold-record") + # The payload opens with a schema version, and the catalog is the store's own description: no recovery. + with self.assertRaises(StoreUnreadableError): + cache.pull_all_backing_stores() + + def _foreign_id_document(self): + return define_database_entry_type( + "tests.foreign-id", + [("name", 0)], + schema_id="tests.foreign-id.current", + schema_name="tests.foreign-id", + schema_version="tests.foreign_id.v1", + compatible_schema_ids=["tests.foreign-id.current", "tests.foreign-id.older"], + ) + + def _open_foreign_id_store(self, document, path: Path) -> CultCache: + cache = CultCache.builder().register_document_type(document).add_generic_store( + SingleFileMessagePackBackingStore(path) + ).build() + cache.pull_all_backing_stores() + return cache + + # A record put under an id its schema only lists as compatible is stamped with the registered id, and the store the write + # leaves opens and takes the next write. + def test_put_envelope_under_a_compatible_schema_id_writes_a_store_that_reopens(self) -> None: + import msgpack # type: ignore + from cultcache_py import CultCacheSchemaCatalogEntry + + document = self._foreign_id_document() + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "store.msgpack" + cache = self._open_foreign_id_store(document, path) + payload = document.encode_payload({"name": "foreign"}) + foreign_entry = CultCacheSchemaCatalogEntry( + schema_id="tests.foreign-id.older", + schema_name="tests.foreign-id", + schema_version="tests.foreign_id.v1", + content_hash="tests.foreign-id.older", + canonical_schema_json="", + compatible_schema_ids=("tests.foreign-id.older",), + members=(), + ) + cache.put_envelope( + document, + CultCacheEnvelope( + key="foreign", + type="tests.foreign-id", + payload=payload, + stored_at="2026-09-30T00:00:00Z", + schema_id="tests.foreign-id.older", + catalog_entry=foreign_entry, + ), + ) + stored = msgpack.unpackb(path.read_bytes(), raw=False) + self.assertEqual([entry[0] for entry in stored[1]], ["tests.foreign-id.current"]) + self.assertEqual(stored[2][0][1], "tests.foreign-id.current") + reopened = self._open_foreign_id_store(document, path) + self.assertEqual(reopened.get_required(document, "foreign")["name"], "foreign") + reopened.put(document, "next", {"name": "next"}) + self.assertEqual(self._open_foreign_id_store(document, path).get_required(document, "next")["name"], "next") + + def test_the_store_publishes_a_records_schema_id_even_when_the_envelopes_catalog_entry_does_not(self) -> None: + import msgpack # type: ignore + from cultcache_py import CultCacheSchemaCatalogEntry + + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "store.msgpack" + store = SingleFileMessagePackBackingStore(path) + store.push(CultCacheEnvelope( + key="k", + type="tests.store-publishes", + payload=msgpack.packb({"name": "k"}, use_bin_type=True), + stored_at="2026-09-30T00:00:00Z", + schema_id="tests.store-publishes.record", + catalog_entry=CultCacheSchemaCatalogEntry( + schema_id="tests.store-publishes.other", + schema_name="tests.store-publishes", + schema_version="tests.store_publishes.v1", + content_hash="tests.store-publishes.other", + canonical_schema_json="", + compatible_schema_ids=("tests.store-publishes.other",), + members=(), + ), + )) + stored = msgpack.unpackb(path.read_bytes(), raw=False) + self.assertEqual(stored[2][0][1], "tests.store-publishes.record") + self.assertEqual([entry[0] for entry in stored[1]], ["tests.store-publishes.record"]) + self.assertEqual(len(SingleFileMessagePackBackingStore(path).pull_all()), 1) + + # The catalog a write leaves is derived from its records: one entry per carried id, chosen and never merged, the same whatever + # order the records arrive in. A registered (supplied) entry wins over an arrived (read back) entry with the same own id. + @staticmethod + def _writer_entry(schema_id: str, name: str, content_hash: str, compatible: tuple[str, ...]): + from cultcache_py import CultCacheSchemaCatalogEntry + + return CultCacheSchemaCatalogEntry( + schema_id=schema_id, schema_name=name, schema_version=name + ".v1", content_hash=content_hash, + canonical_schema_json="", compatible_schema_ids=compatible, members=(), + ) + + def _writer_record(self, key: str, type: str, schema_id: str, catalog_entry=None) -> CultCacheEnvelope: + import msgpack # type: ignore + + return CultCacheEnvelope( + key=key, type=type, payload=msgpack.packb({"key": key}, use_bin_type=True), + stored_at="2026-09-30T00:00:00Z", schema_id=schema_id, catalog_entry=catalog_entry, + ) + + @staticmethod + def _raw_entry(entry) -> list: + return [entry.schema_id, entry.schema_name, entry.schema_version, entry.content_hash, "", list(entry.compatible_schema_ids), []] + + def test_a_registered_entry_is_written_as_it_is_over_the_arrived_entries_with_the_same_own_id_in_either_order(self) -> None: + import msgpack # type: ignore + + stale1 = self._writer_entry("tests.x", "tests.n", "a-stale-1", ("tests.x", "old")) + stale2 = self._writer_entry("tests.x", "tests.n", "a-stale-2", ("tests.x",)) + registered = self._writer_record("a", "tests.n", "tests.x", self._writer_entry("tests.x", "tests.n", "fresh", ("tests.x",))) + for entries in ([stale1, stale2], [stale2, stale1]): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "store.msgpack" + path.write_bytes(msgpack.packb([ + "cultcache.store.v1", [self._raw_entry(entry) for entry in entries], + [["b", "tests.x", "2026-09-30T00:00:00Z", msgpack.packb({"b": 1}, use_bin_type=True)]], + ], use_bin_type=True)) + SingleFileMessagePackBackingStore(path).push(registered) + catalog = msgpack.unpackb(path.read_bytes(), raw=False)[1] + self.assertEqual(len(catalog), 1) + self.assertEqual(catalog[0][3], "fresh") + self.assertEqual(catalog[0][5], ["tests.x"], "written as chosen, no union with the arrived entries' ids") + + def test_a_rename_with_a_stable_schema_id_a_whole_view_write_names_the_registered_schema(self) -> None: + import msgpack # type: ignore + + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "store.msgpack" + path.write_bytes(msgpack.packb([ + "cultcache.store.v1", [self._raw_entry(self._writer_entry("tests.x", "tests.old", "stale", ("tests.x",)))], + [["a", "tests.x", "2026-09-30T00:00:00Z", msgpack.packb({"a": 1}, use_bin_type=True)]], + ], use_bin_type=True)) + SingleFileMessagePackBackingStore(path).push_all( + [self._writer_record("a", "tests.old", "tests.x", self._writer_entry("tests.x", "tests.new", "fresh", ("tests.x",)))] + ) + catalog = msgpack.unpackb(path.read_bytes(), raw=False)[1] + self.assertEqual([(entry[1], entry[3]) for entry in catalog], [("tests.new", "fresh")]) + + def test_two_entries_of_one_tier_that_share_an_own_id_and_disagree_on_the_schema_name_refuse_the_write_typed(self) -> None: + from cultcache_py import SchemaConflictError + + first = self._writer_record("b", "tests.first", "tests.x", self._writer_entry("tests.x", "tests.first", "h1", ("tests.x",))) + second = self._writer_record("a", "tests.first", "tests.x", self._writer_entry("tests.x", "tests.second", "h2", ("tests.x",))) + for order in ([first, second], [second, first]): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "store.msgpack" + with self.assertRaises(SchemaConflictError) as refused: + SingleFileMessagePackBackingStore(path).push_all(order) + self.assertEqual(refused.exception.schema_id, "tests.x") + self.assertEqual(refused.exception.record_key, "a") + self.assertEqual(refused.exception.schema_names, ["tests.first", "tests.second"]) + self.assertFalse(path.exists()) + + def test_records_of_different_types_under_one_schema_id_refuse_the_write_typed_and_no_record_changes_type(self) -> None: + from cultcache_py import SchemaConflictError + + a = self._writer_record("a", "tests.n", "tests.y", self._writer_entry("tests.x", "tests.n", "h1", ("tests.x", "tests.y"))) + b = self._writer_record("b", "tests.m", "tests.y", self._writer_entry("tests.y", "tests.m", "h2", ("tests.y",))) + for order in ([a, b], [b, a]): + with tempfile.TemporaryDirectory() as tmp: + with self.assertRaises(SchemaConflictError) as refused: + SingleFileMessagePackBackingStore(Path(tmp) / "store.msgpack").push_all(order) + self.assertEqual(refused.exception.schema_id, "tests.y") + self.assertEqual(refused.exception.schema_names, ["tests.m", "tests.n"]) + + def test_an_entry_that_owns_an_id_is_chosen_over_one_that_lists_it(self) -> None: + import msgpack # type: ignore + + owner = self._writer_record("a", "tests.n", "tests.y", self._writer_entry("tests.y", "tests.n", "h1", ("tests.y",))) + lister = self._writer_record("b", "tests.n", "tests.x", self._writer_entry("tests.x", "tests.n", "h2", ("tests.x", "tests.y"))) + for order in ([lister, owner], [owner, lister]): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "store.msgpack" + SingleFileMessagePackBackingStore(path).push_all(order) + catalog = msgpack.unpackb(path.read_bytes(), raw=False)[1] + self.assertEqual(sorted((entry[0], entry[5]) for entry in catalog), [("tests.x", ["tests.x", "tests.y"]), ("tests.y", ["tests.y"])]) + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "store.msgpack" + SingleFileMessagePackBackingStore(path).push_all([owner]) + self.assertEqual([entry[0] for entry in msgpack.unpackb(path.read_bytes(), raw=False)[1]], ["tests.y"]) + + def test_entries_of_one_tier_that_tie_are_taken_in_one_fixed_order_and_a_registered_lister_beats_an_arrived_one(self) -> None: + import msgpack # type: ignore + + # The content hash orders them against the order their compatible ids would give: h1 sorts first, its list sorts last. + one = self._writer_record("a", "tests.n", "tests.x", self._writer_entry("tests.x", "tests.n", "h2", ("tests.x",))) + two = self._writer_record("b", "tests.n", "tests.x", self._writer_entry("tests.x", "tests.n", "h1", ("tests.x", "tests.y"))) + for order in ([one, two], [two, one]): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "store.msgpack" + SingleFileMessagePackBackingStore(path).push_all(order) + self.assertEqual([entry[3] for entry in msgpack.unpackb(path.read_bytes(), raw=False)[1]], ["h1"]) + + # z sits under old, which the arrived entry (id x) lists; the supplied entry (id r) lists it too: the registered one publishes it. + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "store.msgpack" + path.write_bytes(msgpack.packb([ + "cultcache.store.v1", [self._raw_entry(self._writer_entry("tests.x", "tests.n", "stale", ("tests.x", "old")))], + [["z", "old", "2026-09-30T00:00:00Z", msgpack.packb({"z": 1}, use_bin_type=True)]], + ], use_bin_type=True)) + SingleFileMessagePackBackingStore(path).push( + self._writer_record("a", "tests.n", "tests.r", self._writer_entry("tests.r", "tests.n", "fresh", ("tests.r", "old"))) + ) + self.assertEqual([entry[0] for entry in msgpack.unpackb(path.read_bytes(), raw=False)[1]], ["tests.r"]) + + # A registered entry that owns an id keeps it against an arrived entry that owns it too and lists the id another record sits + # under, whichever of the two ids sorts first: the arrived entry's list is not merged in, so the record under the listed id is + # published by no chosen entry. + def test_an_arrived_entry_that_owns_a_registered_id_does_not_replace_it_nor_publish_what_it_lists(self) -> None: + import msgpack # type: ignore + from cultcache_py import SchemaConflictError + + for own_id, listed_id in (("tests.a", "tests.b"), ("tests.b", "tests.a")): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "store.msgpack" + path.write_bytes(msgpack.packb([ + "cultcache.store.v1", [self._raw_entry(self._writer_entry(own_id, "tests.arrived", "stale", (own_id, listed_id)))], + [["k2", listed_id, "2026-09-30T00:00:00Z", msgpack.packb({"k": 2}, use_bin_type=True)]], + ], use_bin_type=True)) + before = path.read_bytes() + with self.assertRaises(SchemaConflictError) as refused: + SingleFileMessagePackBackingStore(path).push( + self._writer_record("k1", "tests.registered", own_id, self._writer_entry(own_id, "tests.registered", "fresh", (own_id,))) + ) + self.assertEqual(refused.exception.schema_id, listed_id) + self.assertEqual(refused.exception.record_key, "k2") + self.assertEqual(path.read_bytes(), before, "the file is left as it was") + + # Ids are taken in sorted order, so which entry survives, and so which id the refusal names, does not depend on the order the + # records arrive in. + def test_the_refusal_names_the_same_id_whatever_order_the_records_arrive_in(self) -> None: + from cultcache_py import SchemaConflictError + + one = self._writer_record("r1", "tests.n", "tests.y", self._writer_entry("tests.x", "tests.n", "h1", ("tests.x", "tests.y"))) + two = self._writer_record("r2", "tests.n", "tests.z", self._writer_entry("tests.x", "tests.n", "h2", ("tests.x", "tests.z"))) + for order in ([one, two], [two, one]): + with tempfile.TemporaryDirectory() as tmp: + with self.assertRaises(SchemaConflictError) as refused: + SingleFileMessagePackBackingStore(Path(tmp) / "store.msgpack").push_all(order) + self.assertEqual(refused.exception.schema_id, "tests.y") + self.assertEqual(refused.exception.record_key, "r1") + + # A write may not retype records already in the file: across tiers the id keeps the type its records resolve to, unless the + # entries of both tiers own it, which is a rename under a stable id and the registered descriptor wins. + def test_a_write_that_would_change_the_type_an_existing_record_resolves_to_refuses_typed_and_leaves_the_file(self) -> None: + import msgpack # type: ignore + from cultcache_py import SchemaConflictError + + cases = [ + ("registered entry lists the id, arrived entry owns it", + self._writer_entry("tests.x", "tests.arrived", "h1", ("tests.x",)), + self._writer_record("a", "tests.registered", "tests.x", self._writer_entry("tests.r", "tests.registered", "h2", ("tests.r", "tests.x")))), + ("registered entry owns the id, arrived entry lists it", + self._writer_entry("tests.r", "tests.arrived", "h1", ("tests.r", "tests.x")), + self._writer_record("a", "tests.registered", "tests.x", self._writer_entry("tests.x", "tests.registered", "h2", ("tests.x",)))), + ] + payload = msgpack.packb({"z": 1}, use_bin_type=True) + with tempfile.TemporaryDirectory() as tmp: + for name, catalog, registered in cases: + path = Path(tmp) / (name.split(",")[0].replace(" ", "-") + ".msgpack") + path.write_bytes(msgpack.packb( + ["cultcache.store.v1", [self._raw_entry(catalog)], [["z", "tests.x", "2026-09-30T00:00:00Z", payload]]], use_bin_type=True)) + before = path.read_bytes() + with self.assertRaises(SchemaConflictError, msg=name) as refused: + SingleFileMessagePackBackingStore(path).push(registered) + self.assertEqual(refused.exception.schema_id, "tests.x", name) + self.assertEqual(sorted(refused.exception.schema_names), ["tests.arrived", "tests.registered"], name) + self.assertEqual(path.read_bytes(), before, name) + + # The same registered record renames the schema when both tiers own the id: it writes, and the reader finds every record. + path = Path(tmp) / "rename.msgpack" + path.write_bytes(msgpack.packb([ + "cultcache.store.v1", [self._raw_entry(self._writer_entry("tests.x", "tests.arrived", "h1", ("tests.x",)))], + [["z", "tests.x", "2026-09-30T00:00:00Z", payload]], + ], use_bin_type=True)) + SingleFileMessagePackBackingStore(path).push( + self._writer_record("a", "tests.registered", "tests.x", self._writer_entry("tests.x", "tests.registered", "h2", ("tests.x",))) + ) + self.assertEqual([(entry[0], entry[1]) for entry in msgpack.unpackb(path.read_bytes(), raw=False)[1]], [("tests.x", "tests.registered")]) + self.assertEqual(len(SingleFileMessagePackBackingStore(path).pull_all()), 2) + + def test_a_record_no_chosen_entry_publishes_refuses_the_write_typed_and_leaves_the_file(self) -> None: + import msgpack # type: ignore + from cultcache_py import SchemaConflictError + + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "store.msgpack" + path.write_bytes(msgpack.packb([ + "cultcache.store.v1", [self._raw_entry(self._writer_entry("tests.x", "tests.legacy", "stale", ("tests.x", "old")))], + [["z", "old", "2026-09-30T00:00:00Z", msgpack.packb({"z": 1}, use_bin_type=True)]], + ], use_bin_type=True)) + before = path.read_bytes() + with self.assertRaises(SchemaConflictError) as refused: + SingleFileMessagePackBackingStore(path).push( + self._writer_record("a", "tests.legacy", "tests.x", self._writer_entry("tests.x", "tests.legacy", "fresh", ("tests.x",))) + ) + self.assertEqual(refused.exception.schema_id, "old") + self.assertEqual(refused.exception.record_key, "z") + self.assertEqual(path.read_bytes(), before) + + def test_an_id_an_entry_owns_names_that_entry_not_one_that_lists_it_as_compatible(self) -> None: + for vector in ("own-id-over-compatible-v3.bin", "compatible-before-owner-v3.bin"): + store = SingleFileMessagePackBackingStore(self._C2A_VECTORS / "readability" / vector) + self.assertEqual([envelope.type for envelope in store.pull_all()], ["vectors.item", "vectors.item"], vector) + + def test_a_record_loaded_under_a_compatible_schema_id_is_written_back_under_the_id_its_catalog_entry_carries(self) -> None: + import msgpack # type: ignore + + document = self._foreign_id_document() + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "store.msgpack" + path.write_bytes(msgpack.packb([ + "cultcache.store.v1", + [["tests.foreign-id.current", "tests.foreign-id", "tests.foreign_id.v1", "tests.foreign-id.current", "", + ["tests.foreign-id.current", "tests.foreign-id.older"], []]], + [["old", "tests.foreign-id.older", "2026-09-30T00:00:00Z", document.encode_payload({"name": "old"})]], + ], use_bin_type=True)) + cache = self._open_foreign_id_store(document, path) + self.assertEqual(cache.get_required_envelope(document, "old").schema_id, "tests.foreign-id.current") + cache.put(document, "next", {"name": "next"}) + reopened = self._open_foreign_id_store(document, path) + self.assertEqual(reopened.get_required(document, "old")["name"], "old") + self.assertEqual(reopened.get_required(document, "next")["name"], "next") + + # The cache restamps a record on load, so a store's writer never has to: a record loaded under an id the registered document + # only lists is held, and written back by a whole write, under the id its catalog entry carries. A document that does not list + # the id a record sits under cannot have that record published by a write that leaves it in place: the put is refused, typed. + def test_a_cache_that_does_not_list_the_id_a_loaded_record_sits_under_is_refused_a_put_and_the_file_is_left(self) -> None: + import msgpack # type: ignore + from cultcache_py import SchemaConflictError + + document = define_database_entry_type( + "tests.foreign-id", [("name", 0)], schema_id="tests.foreign-id.current", + schema_name="tests.foreign-id", schema_version="tests.foreign_id.v1", + ) + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "store.msgpack" + path.write_bytes(msgpack.packb([ + "cultcache.store.v1", + [["tests.foreign-id.current", "tests.foreign-id", "tests.foreign_id.v1", "tests.foreign-id.current", "", + ["tests.foreign-id.current", "tests.foreign-id.older"], []]], + [["old", "tests.foreign-id.older", "2026-09-30T00:00:00Z", document.encode_payload({"name": "old"})]], + ], use_bin_type=True)) + before = path.read_bytes() + cache = self._open_foreign_id_store(document, path) + self.assertEqual(cache.get_required_envelope(document, "old").schema_id, "tests.foreign-id.current") + with self.assertRaises(SchemaConflictError) as refused: + cache.put(document, "next", {"name": "next"}) + self.assertEqual(refused.exception.schema_id, "tests.foreign-id.older") + self.assertEqual(refused.exception.record_key, "old") + self.assertEqual(path.read_bytes(), before) + + # A record resolves to a local document by its schema id, never by the name its catalog entry carries: a schema renamed under + # a stable id opens, and an id that names one document is not overruled by a name that names another. + def _store_of(self, path: Path, catalog: list, records: list) -> None: + import msgpack # type: ignore + + path.write_bytes(msgpack.packb(["cultcache.store.v1", catalog, records], use_bin_type=True)) + + def test_a_schema_renamed_under_a_stable_id_opens_by_its_id(self) -> None: + document = define_database_entry_type( + "tests.old", [("name", 0)], schema_id="tests.stable", schema_name="tests.old", schema_version="tests.old.v1", + ) + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "store.msgpack" + self._store_of( + path, + [["tests.stable", "tests.renamed", "tests.renamed.v1", "renamed", "", ["tests.stable"], []]], + [["a", "tests.stable", "2026-09-30T00:00:00Z", document.encode_payload({"name": "a"})]], + ) + cache = self._open_foreign_id_store(document, path) + self.assertEqual(cache.get_required(document, "a")["name"], "a") + + def test_a_schema_id_that_names_one_document_is_not_overruled_by_a_name_that_names_another(self) -> None: + first = define_database_entry_type("tests.a", [("name", 0)], schema_id="id.a", schema_name="tests.a", schema_version="tests.a.v1") + second = define_database_entry_type( + "tests.b", [("name", 0)], schema_id="id.b", schema_name="tests.b", schema_version="tests.b.v1", compatible_schema_ids=["id.b", "id.b.older"], + ) + with tempfile.TemporaryDirectory() as tmp: + for schema_id in ("id.b", "id.b.older"): + path = Path(tmp) / (schema_id + ".msgpack") + # The entry that publishes the id carries the other document's name. + self._store_of( + path, + [["id.b", "tests.a", "tests.b.v1", "h", "", ["id.b", "id.b.older"], []]], + [["k", schema_id, "2026-09-30T00:00:00Z", second.encode_payload({"name": "k"})]], + ) + cache = CultCache.builder().register_document_type(first).register_document_type(second).add_generic_store( + SingleFileMessagePackBackingStore(path) + ).build() + cache.pull_all_backing_stores() + self.assertEqual(cache.get_required(second, "k")["name"], "k", schema_id) + self.assertIsNone(cache.get(first, "k"), schema_id) + + def _assert_second_claim_refused(self, first, second, schema_id: str) -> None: + """A registration refusal is typed, names the id and both schema names and document types, and the cache keeps what it + had.""" + from cultcache_py import SchemaConflictError + + cache = CultCache() + cache.register_document_type(first) + with self.assertRaises(SchemaConflictError) as refused: + cache.register_document_type(second) + self.assertEqual(refused.exception.schema_id, schema_id) + self.assertEqual(refused.exception.schema_names, [first.catalog_entry().schema_name, second.catalog_entry().schema_name]) + self.assertIn(first.type, str(refused.exception)) + self.assertIn(second.type, str(refused.exception)) + if second is not first: + with self.assertRaisesRegex(CultCacheError, "not registered"): + cache.put(second, "k", {"name": "k"}) + + def _pull(self, documents: list, catalog: list, records: list) -> CultCache: + tmp = tempfile.mkdtemp() + path = Path(tmp) / "store.msgpack" + self._store_of(path, catalog, records) + builder = CultCache.builder() + for document in documents: + builder = builder.register_document_type(document) + cache = builder.add_generic_store(SingleFileMessagePackBackingStore(path)).build() + cache.pull_all_backing_stores() + return cache + + def test_a_document_listing_an_id_another_owns_registers_beside_it_in_either_order_and_the_owner_wins(self) -> None: + owner = define_database_entry_type("tests.owner", [("name", 0)], schema_id="id.owned", schema_name="tests.owner", schema_version="tests.owner.v1") + lister = define_database_entry_type( + "tests.lister", [("name", 0)], schema_id="id.lister", schema_name="tests.lister", schema_version="tests.lister.v1", + compatible_schema_ids=["id.lister", "id.owned"], + ) + catalog = [ + ["id.owned", "tests.owner", "tests.owner.v1", "h1", "", ["id.owned"], []], + ["id.lister", "tests.lister", "tests.lister.v1", "h2", "", ["id.lister", "id.owned"], []], + ] + records = [ + ["o", "id.owned", "2026-09-30T00:00:00Z", owner.encode_payload({"name": "o"})], + ["l", "id.lister", "2026-09-30T00:00:00Z", lister.encode_payload({"name": "l"})], + ] + for order in ([owner, lister], [lister, owner]): + cache = self._pull(order, catalog, records) + self.assertEqual(cache.get_required(owner, "o")["name"], "o") + self.assertIsNone(cache.get(lister, "o")) + self.assertEqual(cache.get_required(lister, "l")["name"], "l") + self.assertEqual(self._pull([lister], catalog, records[:1]).get_required(lister, "o")["name"], "o") + + def test_two_documents_listing_one_id_register_and_a_record_under_it_needs_an_owner(self) -> None: + from cultcache_py import SchemaConflictError + + a = define_database_entry_type( + "tests.list-a", [("name", 0)], schema_id="id.a", schema_name="tests.list-a", compatible_schema_ids=["id.a", "id.shared"], + ) + b = define_database_entry_type( + "tests.list-b", [("name", 0)], schema_id="id.b", schema_name="tests.list-b", compatible_schema_ids=["id.b", "id.shared"], + ) + owner = define_database_entry_type("tests.shared", [("name", 0)], schema_id="id.shared", schema_name="tests.shared") + catalog = [["id.shared", "tests.shared", "tests.shared.v1", "h", "", ["id.shared"], []]] + records = [["k", "id.shared", "2026-09-30T00:00:00Z", owner.encode_payload({"name": "k"})]] + for listers in ([a, b], [b, a]): + with self.assertRaises(SchemaConflictError) as refused: + self._pull(listers, catalog, records) + self.assertEqual(refused.exception.schema_id, "id.shared") + self.assertEqual(refused.exception.record_key, "k") + self.assertEqual(refused.exception.schema_names, ["tests.list-a", "tests.list-b"]) + for documents in ([*listers, owner], [owner, *listers]): + cache = self._pull(documents, catalog, records) + self.assertEqual(cache.get_required(owner, "k")["name"], "k") + + def test_two_documents_owning_one_schema_id_are_refused_in_either_order(self) -> None: + # Each declares an older id and not its own, so the id they share is only ever their own. + a = define_database_entry_type("tests.own-a", [("name", 0)], schema_id="id.same", schema_name="tests.own-a", compatible_schema_ids=["id.a-older"]) + b = define_database_entry_type("tests.own-b", [("name", 0)], schema_id="id.same", schema_name="tests.own-b", compatible_schema_ids=["id.b-older"]) + self._assert_second_claim_refused(a, b, "id.same") + self._assert_second_claim_refused(b, a, "id.same") + + def test_every_registration_refusal_is_a_schema_conflict(self) -> None: + first = define_database_entry_type("tests.named", [("name", 0)], schema_id="id.first", schema_name="tests.named") + same_name = define_database_entry_type("tests.renamed", [("name", 0)], schema_id="id.second", schema_name="tests.named") + self._assert_second_claim_refused(first, same_name, "id.second") + self._assert_second_claim_refused(first, first, "id.first") def test_interop_cli_helpers_round_trip_v1_store(self) -> None: with tempfile.TemporaryDirectory() as tmp: @@ -893,6 +1373,32 @@ class CultCacheTests(unittest.TestCase): self.assertNotIn(repr(value[0]), message) self.assertIn(f"of {len(value[0])} bytes", message) + def test_single_file_refusals_say_exactly_what_was_found_and_what_is_read(self) -> None: + import msgpack # type: ignore + + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "store.cc" + reads = "it reads 'cultcache.store.v1' and 'cultcache.store.v3' only" + for header, found in ( + ("cultcache.store.v12", "'cultcache.store.v12'"), + ("cultcache.store.SECRET", "an unrecognised cultcache.store.* header of 22 bytes"), + ("cultcache.store.v", "an unrecognised cultcache.store.* header of 17 bytes"), + # Digits are ASCII only: two Arabic-Indic digits are four bytes and are not the known shape. + ("cultcache.store.v١٢", "an unrecognised cultcache.store.* header of 21 bytes"), + ): + path.write_bytes(msgpack.packb([header, [], []], use_bin_type=True)) + with self.assertRaises(ValueError) as caught: + SingleFileMessagePackBackingStore(path).pull_all() + self.assertIn(f"CultCache store format {found} is not one this runtime reads; {reads}", str(caught.exception)) + + path.write_bytes(msgpack.packb(["cultcache.store.v1", [], [["k", "s", "t", b"\x90", "extra"]]], use_bin_type=True)) + with self.assertRaises(ValueError) as caught: + SingleFileMessagePackBackingStore(path).pull_all() + self.assertIn( + "CultCache record 'k' (schema 's') has 5 slots, more than the 4 of a cultcache.store.v1 record, so this is not a valid store", + str(caught.exception), + ) + def test_single_file_record_refusal_names_only_a_string_key(self) -> None: import msgpack # type: ignore @@ -966,5 +1472,120 @@ class CultCacheTests(unittest.TestCase): self.assertNotIsInstance(caught.exception, FileNotFoundError) + # The element-id marker: tests/vectors/document-variants-c2a/v3-base.msgpack is v1-base with its header replaced. + _C2A_VECTORS = Path(__file__).resolve().parents[3] / "tests" / "vectors" / "document-variants-c2a" + + def _header_after_rewrite(self, vector: Path) -> str: + import msgpack + + with tempfile.TemporaryDirectory() as tmp: + store_path = Path(tmp) / "store.msgpack" + store_path.write_bytes(vector.read_bytes()) + store = SingleFileMessagePackBackingStore(store_path) + envelopes = store.pull_all() + self.assertEqual([e.key for e in envelopes], ["alpha", "beta"]) + store.push(envelopes[0]) + return msgpack.unpackb(store_path.read_bytes(), raw=False)[0] + + def test_single_file_reads_a_v3_element_id_store_and_a_rewrite_keeps_the_marker(self) -> None: + self.assertEqual(self._header_after_rewrite(self._C2A_VECTORS / "v3-base.msgpack"), "cultcache.store.v3") + + def test_single_file_rewrite_of_a_v1_store_stays_v1(self) -> None: + self.assertEqual(self._header_after_rewrite(self._VECTORS / "v1-base.msgpack"), "cultcache.store.v1") + + def test_single_file_flush_after_the_marked_file_is_gone_writes_unmarked(self) -> None: + import msgpack + + with tempfile.TemporaryDirectory() as tmp: + store_path = Path(tmp) / "store.msgpack" + store_path.write_bytes((self._C2A_VECTORS / "v3-base.msgpack").read_bytes()) + store = SingleFileMessagePackBackingStore(store_path) + envelopes = store.pull_all() + store_path.unlink() + store.push_all(envelopes) + self.assertEqual(msgpack.unpackb(store_path.read_bytes(), raw=False)[0], "cultcache.store.v1") + + def test_single_file_push_after_the_marked_file_is_gone_writes_unmarked(self) -> None: + import msgpack + + with tempfile.TemporaryDirectory() as tmp: + store_path = Path(tmp) / "store.msgpack" + store_path.write_bytes((self._C2A_VECTORS / "v3-base.msgpack").read_bytes()) + store = SingleFileMessagePackBackingStore(store_path) + envelopes = store.pull_all() + store_path.unlink() + store.push(envelopes[0]) + self.assertEqual(msgpack.unpackb(store_path.read_bytes(), raw=False)[0], "cultcache.store.v1") + + def test_single_file_push_all_refuses_a_store_whose_header_it_cannot_read_and_leaves_it_untouched(self) -> None: + envelopes = SingleFileMessagePackBackingStore(self._C2A_VECTORS / "v3-base.msgpack").pull_all() + vectors = self._VECTORS.parent + for vector in ( + self._VECTORS / "unknown-header.msgpack", + self._VECTORS / "variant-v2.msgpack", + vectors / "document-variants-c1" / "variant-store.msgpack", + ): + with tempfile.TemporaryDirectory() as tmp: + store_path = Path(tmp) / "store.msgpack" + store_path.write_bytes(vector.read_bytes()) + before = store_path.read_bytes() + with self.assertRaisesRegex(StoreUnreadableError, "is not readable"): + SingleFileMessagePackBackingStore(store_path).push_all(envelopes) + self.assertEqual(store_path.read_bytes(), before, f"{vector.name} was rewritten") + + # A file is replaced by a rewrite exactly when this runtime's own reader opens it: one verdict per file, asked by + # pull_all, push_all, push and delete alike. The bytes and every runtime's verdict: + # tests/vectors/document-variants-c2a/readability. + def test_single_file_replaces_a_file_exactly_when_it_reads(self) -> None: + import msgpack # type: ignore + + root = self._C2A_VECTORS / "readability" + rows = [ + line.split() + for line in (root / "manifest.txt").read_text(encoding="utf-8").splitlines() + if line and not line.startswith("#") + ] + # Every vector in the folder has a manifest row. + self.assertEqual( + sorted(path.name for path in root.glob("*.bin")), + sorted(vector for vector, *_ in rows if not vector.startswith("..")), + ) + envelopes = SingleFileMessagePackBackingStore(self._C2A_VECTORS / "v3-base.msgpack").pull_all() + for vector, _, _, _, python in rows: + content = (root / vector).read_bytes() + for operation in ("pull_all", "push_all", "push", "delete"): + with tempfile.TemporaryDirectory() as tmp: + store_path = Path(tmp) / "store.msgpack" + store_path.write_bytes(content) + store = SingleFileMessagePackBackingStore(store_path) + run = { + "pull_all": store.pull_all, + "push_all": lambda: store.push_all(envelopes), + "push": lambda: store.push(envelopes[0]), + "delete": lambda: store.delete(envelopes[0].type, envelopes[0].key), + }[operation] + if python == "reads": + run() + if operation == "pull_all": + continue + # The replaced file is a store: it carries the header its old content decides (a v3 store keeps its + # marker, all else is v1), and it holds what the operation wrote. + expected = "cultcache.store.v3" if "v3" in vector else "cultcache.store.v1" + self.assertEqual(msgpack.unpackb(store_path.read_bytes(), raw=False)[0], expected, f"{vector} {operation}") + keys = [envelope.key for envelope in SingleFileMessagePackBackingStore(store_path).pull_all()] + if operation == "push_all": + self.assertEqual(sorted(set(keys) & {e.key for e in envelopes}), sorted(e.key for e in envelopes), f"{vector} {operation}") + elif operation == "push": + self.assertIn(envelopes[0].key, keys, f"{vector} {operation}") + else: + self.assertNotIn(envelopes[0].key, keys, f"{vector} {operation}") + else: + with self.assertRaises(StoreUnreadableError, msg=f"{vector} {operation}") as refused: + run() + self.assertIsNotNone(refused.exception.__cause__, f"{vector} {operation}") + self.assertEqual(store_path.read_bytes(), content, f"{vector} {operation} rewrote a file it cannot read") + + + if __name__ == "__main__": unittest.main() diff --git a/packages/cultcache-py/tests/test_typed_leaves.py b/packages/cultcache-py/tests/test_typed_leaves.py index 9b432efd..b284bcbb 100644 --- a/packages/cultcache-py/tests/test_typed_leaves.py +++ b/packages/cultcache-py/tests/test_typed_leaves.py @@ -133,6 +133,7 @@ def _inner_record() -> t.SlotCodec: [ database_entry_field("n", 0, default="", codec=t.string), database_entry_field("w", 1, default=0.0, codec=t.float32), + database_entry_field("id", 2, default="", codec=t.string), ], type_name=REFERENCE + "LeafInner", ) @@ -662,7 +663,7 @@ def _python_authored_leaf_record() -> dict: return { "id": "py", "flag": False, "blob": bytearray([1, 255]), "i": -(2**31), "l": 2**63 - 1, "f": 0.1, "d": 1e-300, "maybe_i": None, "tags": [], "names": ("\u00e9", "z"), "any": {"k": [1, True, None, 2.5, "s"], "n": -3}, - "nested": None, "items": [{"n": "one", "w": 2**53 + 2**29 + 1}, {"n": "two", "w": 1e-40}], + "nested": None, "items": [{"n": "one", "w": 2**53 + 2**29 + 1, "id": "000000000021"}, {"n": "two", "w": 1e-40, "id": "000000000022"}], "shape": UnionArm("LeafCircle", {"r": 0.1}), "floats": [0.1, 3, -0.0, 1e39], "maybe_f": None, } diff --git a/packages/cultcache-py/tests/test_union_parity.py b/packages/cultcache-py/tests/test_union_parity.py index e6fe6c2f..451cd030 100644 --- a/packages/cultcache-py/tests/test_union_parity.py +++ b/packages/cultcache-py/tests/test_union_parity.py @@ -68,22 +68,51 @@ image = define_record_codec( ], type_name=_GOLDEN + "Image", ) -wrap = define_record_codec("Wrap", [database_entry_field("inner", 0, codec=_node)], type_name=_GOLDEN + "Wrap") +wrap = define_record_codec( + "Wrap", + [database_entry_field("inner", 0, codec=_node), database_entry_field("id", 1, codec=t.string)], + type_name=_GOLDEN + "Wrap", +) node = define_union( "Node", {0: ("Box", box), 1: ("Text", text), 2: ("Use", use), 5: ("Image", image), 6: ("Wrap", wrap)}, type_name=_GOLDEN + "Node", ) +# The golden's Outcome: the arms of a list carry an element id, so Denied, Floor and Ceiling are one-slot records. +_id_only = lambda name: define_record_codec(name, [database_entry_field("id", 0, codec=t.string)]) # noqa: E731 outcome = define_union( + "Outcome", + { + 0: ( + "Pending", + define_record_codec( + "Pending", + [database_entry_field("message", 0, codec=t.string), database_entry_field("id", 1, codec=t.string)], + ), + ), + 1: ( + "Completed", + define_record_codec( + "Completed", + [database_entry_field("source_version", 0, codec=t.int32), database_entry_field("id", 1, codec=t.string)], + ), + ), + 2: ("Denied", _id_only("Denied")), + -2147483648: ("Floor", _id_only("Floor")), + 2147483647: ("Ceiling", _id_only("Ceiling")), + }, + type_name=_GOLDEN + "Outcome", +) +# The codec's unit arms (no value, written [key, []]), kept apart from the golden: a document list's arms carry ids. +unit_outcome = define_union( "Outcome", { 0: ("Pending", define_record_codec("Pending", [database_entry_field("message", 0, codec=t.string)])), - 1: ("Completed", define_record_codec("Completed", [database_entry_field("source_version", 0, codec=t.int32)])), 2: ("Denied", None), -2147483648: ("Floor", None), 2147483647: ("Ceiling", None), }, - type_name=_GOLDEN + "Outcome", + type_name=_GOLDEN + "UnitOutcome", ) tool = define_record_codec( "Tool", @@ -121,41 +150,41 @@ def _surface_value() -> dict: "root": UnionArm( "Box", { - "id": "root", + "id": "0000000000c0", "children": [ - UnionArm("Text", {"id": "t1", "content": "hello"}), + UnionArm("Text", {"id": "0000000000c1", "content": "hello"}), UnionArm( "Use", { - "id": "u1", + "id": "0000000000c2", "composition": { "name": "comp", "body": UnionArm( "Box", { - "id": "inner", + "id": "0000000000d0", "children": [ - UnionArm("Text", {"id": "deep", "content": "x"}), - UnionArm("Image", {"id": "img", "uri": "uri://a", "alt": "alt"}), + UnionArm("Text", {"id": "0000000000d1", "content": "x"}), + UnionArm("Image", {"id": "0000000000d2", "uri": "uri://a", "alt": "alt"}), ], }, ), }, "slots": [ - UnionArm("Text", {"id": "s1", "content": "slot"}), + UnionArm("Text", {"id": "0000000000e1", "content": "slot"}), None, - UnionArm("Box", {"id": "b2", "children": []}), + UnionArm("Box", {"id": "0000000000e2", "children": []}), ], }, ), - UnionArm("Image", {"id": "i2", "uri": "uri://b", "alt": ""}), + UnionArm("Image", {"id": "0000000000c3", "uri": "uri://b", "alt": ""}), ], }, ), "outcomes": [ - UnionArm("Pending", {"message": "wait"}), - UnionArm("Completed", {"source_version": 7}), - UnionArm("Denied"), + UnionArm("Pending", {"message": "wait", "id": "0000000000a1"}), + UnionArm("Completed", {"source_version": 7, "id": "0000000000a2"}), + UnionArm("Denied", {"id": "0000000000a3"}), ], "part": UnionArm("Tool", {"name": "hammer", "uses": 12}), "crafted": UnionArm("Part", {"name": "gear", "count": 4}), @@ -256,12 +285,14 @@ class UnionCodecRuleTests(unittest.TestCase): ) def test_unit_arm_is_written_with_empty_slots_and_reads_back(self) -> None: - self.assertEqual(outcome.encode(UnionArm("Denied")), [2, []]) - self.assertEqual(outcome.decode([2, []]), UnionArm("Denied", None)) + # The bytes are the C# reference's (tests/vectors/union-parity/unit-arm-denied.msgpack), not spelled here. + reference = _vector("unit-arm-denied.msgpack") + self.assertEqual(msgpack.packb(unit_outcome.encode(UnionArm("Denied"))), reference) + self.assertEqual(unit_outcome.decode(msgpack.unpackb(reference, raw=False)), UnionArm("Denied", None)) def test_unit_arm_refuses_a_value_to_carry(self) -> None: with self.assertRaises(ValueError) as caught: - outcome.encode(UnionArm("Denied", {"message": self.CANARY})) + unit_outcome.encode(UnionArm("Denied", {"message": self.CANARY})) self.assertEqual(str(caught.exception), "Union 'Outcome' arm 2 is a unit arm and carries no value") def test_arm_keys_are_written_as_declared(self) -> None: @@ -333,7 +364,7 @@ class UnionCodecRuleTests(unittest.TestCase): self.assertEqual(row.decode(encoded), {"id": "r", "cells": cells}) def test_a_unit_arm_with_extra_slots_is_read_past(self) -> None: - self.assertEqual(outcome.decode([2, ["extra"]]), UnionArm("Denied")) + self.assertEqual(unit_outcome.decode([2, ["extra"]]), UnionArm("Denied")) def test_union_arms_are_immutable_hashable_values(self) -> None: arm = UnionArm("Denied") @@ -478,7 +509,7 @@ CANARY = "CANARY-union-golden-4c2a" def _wrap_chain(wraps: int, leaf: UnionArm) -> UnionArm: value = leaf for _ in range(wraps): - value = UnionArm("Wrap", {"inner": value}) + value = UnionArm("Wrap", {"inner": value, "id": "0000000000b1"}) return value @@ -521,9 +552,9 @@ class BoundsVectorTests(unittest.TestCase): self.assertEqual(counts, {"accepted": 4, "refused": 7, "read-only": 1}) def test_the_value_at_the_limit_writes_the_vector_and_the_value_past_it_is_refused_on_write(self) -> None: - at_limit = {**_surface_value(), "root": _wrap_chain(126, UnionArm("Box", {"id": "leaf", "children": []}))} + at_limit = {**_surface_value(), "root": _wrap_chain(126, UnionArm("Box", {"id": "0000000000f0", "children": []}))} self.assertEqual(surface_document.encode_payload(at_limit), _vector("depth-at-limit.msgpack")) - past = {**_surface_value(), "root": _wrap_chain(127, UnionArm("Text", {"id": "leaf", "content": CANARY}))} + past = {**_surface_value(), "root": _wrap_chain(127, UnionArm("Text", {"id": "0000000000f0", "content": CANARY}))} with self.assertRaises(CultCachePayloadDepthError) as caught: surface_document.encode_payload(past) message = str(caught.exception) @@ -533,7 +564,7 @@ class BoundsVectorTests(unittest.TestCase): self.assertNotIn(CANARY, message) def test_a_cache_put_of_the_value_past_the_limit_refuses_and_stores_nothing(self) -> None: - past = {**_surface_value(), "root": _wrap_chain(127, UnionArm("Box", {"id": "leaf", "children": []}))} + past = {**_surface_value(), "root": _wrap_chain(127, UnionArm("Box", {"id": "0000000000f0", "children": []}))} with tempfile.TemporaryDirectory() as tmp: path = Path(tmp) / "store.cc" cache = ( @@ -709,8 +740,12 @@ class UnionDecodeBoundsTests(unittest.TestCase): self.assertEqual(str(caught.exception), expected) def test_the_ends_of_the_int32_key_domain_are_unit_arms_that_read_and_write(self) -> None: - self.assertEqual(outcome.decode([-(2**31), []]), UnionArm("Floor")) - self.assertEqual(outcome.encode(UnionArm("Ceiling")), [2**31 - 1, []]) + floor = _vector("unit-arm-floor.msgpack") + ceiling = _vector("unit-arm-ceiling.msgpack") + self.assertEqual(unit_outcome.decode(msgpack.unpackb(floor, raw=False)), UnionArm("Floor")) + self.assertEqual(msgpack.packb(unit_outcome.encode(UnionArm("Floor"))), floor) + self.assertEqual(unit_outcome.decode(msgpack.unpackb(ceiling, raw=False)), UnionArm("Ceiling")) + self.assertEqual(msgpack.packb(unit_outcome.encode(UnionArm("Ceiling"))), ceiling) if __name__ == "__main__": diff --git a/packages/cultcache-rs/CHANGELOG.md b/packages/cultcache-rs/CHANGELOG.md index 4fa680b9..f15fcee2 100644 --- a/packages/cultcache-rs/CHANGELOG.md +++ b/packages/cultcache-rs/CHANGELOG.md @@ -4,6 +4,17 @@ All notable changes to `cultcache-rs` are recorded here, following [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). Earlier releases are recorded only in git history. +## Unreleased + +### Breaking + +- A single-file store written as `cultcache.store.v3` is read and written: a record that holds an element id (an owned list element with an `[CultElementId]` member, carried through every write) marks the store v3, and a v3 store is carried through a rewrite with its ids. A runtime from before this change refuses a v3 store by name. +- A write over a file the reader refuses (a zero-byte file, a truncated store, a stamp this runtime does not read) refuses with `CultCacheStoreUnreadable` and leaves the file as it is. A whole-store write used to replace such a file. This reverses 0.3.1, where `push_all` replaced a file with no readable stamp (healing it); the stamp rule now has one owner, the reader every write asks under the lock, and it accepts v1 and v3 (0.3.1 refused v3). + +### Added + +- `SchemaConflictError`, the typed refusal of a write the catalog cannot describe: records of different types under one schema id, or an arrived schema and a registered one that share an id and disagree on the schema name. A whole-store write (`push_all`) refuses with it, names the schema id, the names involved and a record key, writes nothing and leaves the file as it is. A caller tells it apart with `error.downcast_ref::()`. + ## [0.3.1] ### Fixed diff --git a/packages/cultcache-rs/examples/cultcache_interop.rs b/packages/cultcache-rs/examples/cultcache_interop.rs index c9bac14d..8efb9395 100644 --- a/packages/cultcache-rs/examples/cultcache_interop.rs +++ b/packages/cultcache-rs/examples/cultcache_interop.rs @@ -26,6 +26,38 @@ struct CultCacheInteropNote { tags: Vec, } +// An object list: each mark is a positional array [label, id], the id an ordinary member at slot 1. +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +struct CultCacheInteropMark(String, String); + +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "cultcache.interop-deck", schema = "CultCacheInteropDeck")] +struct CultCacheInteropDeck { + #[cultcache(key = 0)] + document_id: String, + #[cultcache(key = 1, default)] + marks: Vec, +} + +fn read_deck(file: &str) -> Result<()> { + let mut cache = CultCache::new(); + cache.register_entry_type::()?; + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(file))?; + cache.pull_all_backing_stores()?; + let decks = cache.get_all::()?; + let deck = decks + .first() + .ok_or_else(|| anyhow!("no cultcache.interop-deck records found"))?; + println!( + "{}", + serde_json::to_string(&json!({ + "documentId": deck.document_id, + "ids": deck.marks.iter().map(|mark| mark.1.clone()).collect::>(), + }))? + ); + Ok(()) +} + fn main() -> Result<()> { let mut args = std::env::args().skip(1); let mode = args @@ -37,6 +69,7 @@ fn main() -> Result<()> { match mode.as_str() { "write" => write_note(file, require_arg(&options, "runtime-id")?)?, "read" => read_note(file)?, + "read-deck" => read_deck(file)?, "hold-lock" => hold_lock(file, require_arg(&options, "ms")?.parse()?)?, _ => return Err(anyhow!("unknown mode {mode}")), } diff --git a/packages/cultcache-rs/src/lib.rs b/packages/cultcache-rs/src/lib.rs index 2818c638..b77f4971 100644 --- a/packages/cultcache-rs/src/lib.rs +++ b/packages/cultcache-rs/src/lib.rs @@ -464,6 +464,11 @@ impl<'de> serde::Deserialize<'de> for PersistedRecord { const STORE_FORMAT_V1: &str = "cultcache.store.v1"; +/// A store that can hold element ids. A reader older than element ids refuses this header: it would skip the id +/// slots of a nested element and rewrite the element without them. This runtime writes it back only for a file it +/// found under it; it does not decide when a document carries ids. +const STORE_FORMAT_ELEMENT_IDS: &str = "cultcache.store.v3"; + /// Opens every refusal CultCache raises about one persisted record. The decoder's own /// messages can quote the bytes; these name only the record's key and schema id. const RECORD_REFUSAL: &str = "CultCache record "; @@ -687,18 +692,27 @@ const DECODE_STACK_SEGMENT: usize = 2 * 1024 * 1024; /// ``` pub const MAX_RECORD_INLINE_BYTES: usize = 16 * 1024; -/// The one grown decoder of stored bytes: the outer stack check, then `serde_stacker` re-checks at -/// every container serde's impls descend through, skipped values included. No nesting bound is applied here; [`decode_payload`] walks first. -fn decode_grown(bytes: &[u8]) -> std::result::Result { +/// The one grown setup for every decode of stored bytes: the outer stack check, then `serde_stacker` +/// re-checks at every container serde's impls descend through, skipped values included. `body` runs on the grown stack with the +/// grown deserializer over `reader`, so a caller that must read `reader`'s position afterwards (the store header) still can. +fn grown( + reader: &mut rmp_serde::Deserializer, + body: impl FnOnce(serde_stacker::Deserializer<&mut rmp_serde::Deserializer>) -> T, +) -> T { stacker::maybe_grow(DECODE_RED_ZONE, DECODE_STACK_SEGMENT, || { - let mut reader = rmp_serde::Deserializer::from_read_ref(bytes); - let mut grown = serde_stacker::Deserializer::new(&mut reader); + let mut grown = serde_stacker::Deserializer::new(reader); grown.red_zone = DECODE_RED_ZONE; grown.stack_size = DECODE_STACK_SEGMENT; - T::deserialize(grown) + body(grown) }) } +/// The one grown decoder of a value from stored bytes. No nesting bound is applied here; [`decode_payload`] walks first. +fn decode_grown(bytes: &[u8]) -> std::result::Result { + let mut reader = rmp_serde::Deserializer::from_read_ref(bytes); + grown(&mut reader, |grown| T::deserialize(grown)) +} + /// The one decode of a document payload: the nesting bound first, then [`decode_grown`], whose /// rmp-serde depth counter (1024) is only a backstop. A failure keeps its kind and never quotes the /// bytes. Readers of stored envelope payloads outside this crate decode through it too. @@ -774,17 +788,98 @@ fn describe_serde_message(message: &str) -> String { } } -/// The store header string, or `None` when the bytes do not open with an array whose -/// first slot is a string (the legacy envelope array starts with a map). Only the -/// leading bytes are read, so a store cut short after its header still names its format. -fn store_header(bytes: &[u8]) -> Option { - let mut offset = 0usize; - if read_array_header(bytes, &mut offset)? == 0 { - return None; - } - read_string(bytes, &mut offset) +/// What one pass over a file's bytes learned about its header. `first` is the leading slot when it is a MessagePack +/// string (`str`, never `bin`, as in every other runtime), and `None` when the bytes do not open with an array whose first +/// slot is a string (a legacy envelope array candidate). `whole` is the proof that the bytes are exactly one complete +/// MessagePack array: a truncated store or bytes after the array fail it, and are never taken for a legacy file. A store cut +/// short after its header string still names its format, so `first` survives a failed `whole` and a newer store is refused +/// as unsupported, not as damaged. +struct StoreHeader { + first: Option, + whole: Result<()>, } +/// The one reader of a store file's header. Nothing else classifies leading bytes. +fn store_header(bytes: &[u8]) -> StoreHeader { + /// A string slot and only a string: MessagePack `bin` is not a header. + fn text<'de, D: serde::Deserializer<'de>>(deserializer: D) -> std::result::Result { + struct TextVisitor; + impl serde::de::Visitor<'_> for TextVisitor { + type Value = String; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("a MessagePack string") + } + + fn visit_str(self, value: &str) -> std::result::Result { + Ok(value.to_string()) + } + } + deserializer.deserialize_any(TextVisitor) + } + + #[derive(serde::Deserialize)] + #[serde(untagged)] + enum FirstSlot { + Text(#[serde(deserialize_with = "text")] String), + Other(serde::de::IgnoredAny), + } + + struct HeaderVisitor<'a>(&'a mut Option); + impl<'de> serde::de::Visitor<'de> for HeaderVisitor<'_> { + type Value = (); + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("an array whose first slot may be the store header") + } + + fn visit_seq>(self, mut seq: A) -> std::result::Result<(), A::Error> { + if let Some(FirstSlot::Text(header)) = seq.next_element::()? { + *self.0 = Some(header); + } + while seq.next_element::()?.is_some() {} + Ok(()) + } + } + + let mut first = None; + let mut cursor = std::io::Cursor::new(bytes); + // Walked on the grown stack, as every other decode of stored bytes is: a store may nest values deeper than the thread allows. + let walked = { + let mut reader = rmp_serde::Deserializer::new(&mut cursor); + grown(&mut reader, |grown| serde::Deserializer::deserialize_seq(grown, HeaderVisitor(&mut first))) + }; + let after = bytes.len() - cursor.position() as usize; + let whole = walked + .map_err(|error| opaque_decode_error(error).context("the store is not one complete MessagePack array")) + .and_then(|()| { + ensure!(after == 0, "the store has {after} bytes after its MessagePack array"); + Ok(()) + }); + StoreHeader { first, whole } +} + +/// A write the catalog cannot describe: records of different types under one schema id. This writer derives one entry per carried +/// schema id from its records, so that is its only conflict. Nothing is written. It names the id, the two names and a record key. +#[derive(Debug)] +pub struct SchemaConflictError { + pub schema_id: String, + pub schema_names: Vec, + pub record_key: String, +} + +impl std::fmt::Display for SchemaConflictError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + formatter, + "schema id {:?} cannot describe record {:?}: {:?}", + self.schema_id, self.record_key, self.schema_names + ) + } +} + +impl std::error::Error for SchemaConflictError {} + /// A single-file store whose bytes were read and not accepted. It sits in the error /// chain of every such read, so a consumer tells it apart with /// `error.downcast_ref::()`. A failure to read the bytes @@ -872,6 +967,8 @@ impl std::error::Error for CultCacheStoreWriteFailed {} #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub struct PushAllOptions { + /// Seed the store only where nothing is: a single-file flush writes when nothing is at the path, leaves a store it can + /// read byte for byte, and refuses what its reader refuses (a dangling link included), as a plain flush refuses. pub soft: bool, } @@ -1243,43 +1340,45 @@ impl SingleFileMessagePackBackingStore { } fn read_all_unlocked(&self) -> Result> { - // Only a store that is not there is empty. A file that is there and empty is - // not a store (no writer leaves one), and any other failure to reach the file, - // such as a symlink loop, a file where a directory should be, or no permission, - // stays an io::Error: reading it as empty would let a writer replace a store it - // never saw. - // A dangling symbolic link is something at the path, not nothing: reading it as - // empty would let the writer rename a file over the link and move the store off - // the volume the link pointed to. + Ok(self.read_store_unlocked()?.envelopes) + } + + /// The one reader of the store file: open, every rewrite and the read-only snapshot ask it, so they agree on which + /// files are stores. Only a file that is not there is an empty store. A file that is there and empty is not a store + /// (no writer leaves one), and any other failure to reach the file, such as a symlink loop, a file where a directory + /// should be, or no permission, stays an io::Error: reading it as empty would let a writer replace a store it never + /// saw. A dangling symbolic link is something at the path, not nothing, for the same reason. Anything else must decode + /// completely, as exactly one store or one legacy envelope array; bytes this runtime cannot read are refused with + /// [`CultCacheStoreUnreadable`], so a rewrite never overwrites what it cannot see. + fn read_store_unlocked(&self) -> Result { let bytes = match fs::read(&self.path) { Ok(bytes) => bytes, Err(error) if error.kind() == std::io::ErrorKind::NotFound && is_absent(&self.path)? => { - return Ok(Vec::new()); + return Ok(DiskStore::absent()); } Err(error) => { return Err(anyhow::Error::new(error) .context(format!("failed to read {}", self.path.display()))); } }; - self.refuse_unread_format(&bytes)?; - if store_header(&bytes).as_deref() == Some(STORE_FORMAT_V1) { - decode_store_snapshot(&bytes) - .map_err(|error| self.unreadable(CultCacheStoreUnreadableKind::Undecodable, error)) - } else { - decode_grown(&bytes) - .context("failed to decode legacy CultCache envelope array") - .map_err(|error| self.unreadable(CultCacheStoreUnreadableKind::Undecodable, error)) - } + let head = store_header(&bytes); + self.refuse_unread_format(head.first.as_deref())?; + head.whole + .and_then(|()| decode_store_file(&bytes, head.first.as_deref())) + .map_err(|error| self.unreadable(CultCacheStoreUnreadableKind::Undecodable, error)) } - // The stamp rule, owned once for the reader and the whole-store writer: a - // cultcache.store.* header other than v1 is a format this runtime does not read. Only - // the header is judged, so a newer store cut short is still a newer store. - fn refuse_unread_format(&self, bytes: &[u8]) -> Result<()> { - match store_header(bytes) { - Some(header) if header != STORE_FORMAT_V1 && header.starts_with("cultcache.store.") => Err(self.unreadable( + // The one owner of the stamp rule, asked only by the one reader: a cultcache.store.* header this runtime does not read + // (it reads v1 and v3) is refused by name, judged from the header alone, so a newer store cut short is still a newer + // store. Every write over a file asks that reader first, so a whole-store write judges a stamp as a read does. + fn refuse_unread_format(&self, header: Option<&str>) -> Result<()> { + match header { + Some(header) if header.starts_with("cultcache.store.") && readable_store_format(header).is_none() => Err(self.unreadable( CultCacheStoreUnreadableKind::UnsupportedFormat, - anyhow!("its header is {}; this runtime reads {STORE_FORMAT_V1:?} only", describe_header(&header)), + anyhow!( + "its header is {}; this runtime reads {STORE_FORMAT_V1:?} and {STORE_FORMAT_ELEMENT_IDS:?} only", + describe_header(header) + ), )), _ => Ok(()), } @@ -1312,7 +1411,18 @@ impl SingleFileMessagePackBackingStore { path: self.path.clone(), kind, }; - let bytes = encode_store_snapshot(entries) + // The header a rewrite writes is the one the file on disk carries: a file marked for element ids stays marked (the + // payloads this store passes through may hold ids), one that is gone or legacy is written v1. The file is read by + // the same reader as an open, so a store this runtime cannot read completely, a variant included, is refused + // and never overwritten. + let disk = match self.read_store_unlocked() { + // A dangling link holds no store, so a whole-file replace has no header to keep and replaces the link (R3 pins + // this). The only NotFound the reader returns is that one: a merging writer has already refused it when it + // read the current entries. + Err(error) if io_not_found(&error) => DiskStore::absent(), + other => other?, + }; + let bytes = encode_store_snapshot(entries, &disk) .and_then(|snapshot| rmp_serde::to_vec(&snapshot).context("failed to encode MessagePack")) .map_err(|error| error.context(failed(CultCacheStoreWriteFailedKind::Rejected)))?; self.replace_unlocked(&bytes) @@ -1581,13 +1691,14 @@ impl CacheBackingStore for SingleFileMessagePackBackingStore { }) } - fn push_all(&mut self, entries: &[CultCacheEnvelope], _options: PushAllOptions) -> Result<()> { + fn push_all(&mut self, entries: &[CultCacheEnvelope], options: PushAllOptions) -> Result<()> { self.with_exclusive_lock(|| { - // A file it cannot read is written as before (a dangling link is replaced, R3; a - // directory is a typed write failure); only a stamp it can read is judged. - if let Ok(bytes) = fs::read(&self.path) { - self.refuse_unread_format(&bytes)?; + // A soft flush asks the one reader: nothing at the path is written, a store it reads is left alone, and anything + // the reader refuses (an unreadable file, a dangling link) is refused here as it is for open. + if options.soft && !self.read_store_unlocked()?.absent { + return Ok(()); } + let mut entries = entries.to_vec(); entries.sort_by_key(entry_id); self.write_all_unlocked(&entries) @@ -2667,6 +2778,7 @@ impl CultCache { let Some(canonical_type) = resolve_registered_type( &known_types, &schema_name_definitions, + entry.schema_id.as_deref(), &entry.r#type, ) else { return Err(anyhow!( @@ -3102,40 +3214,68 @@ fn now_utc_second() -> String { chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true) } -fn encode_store_snapshot(entries: &[CultCacheEnvelope]) -> Result { - let mut schema_types = BTreeMap::::new(); +/// The catalog names each schema id once. An entry the file already holds is arrived, and its type is its schema's name; +/// one the caller supplies is registered. Records of one tier under one id are one type. Across tiers a write may not +/// retype what the file holds, unless the registered type owns the id (a registered type's id is its entry type): that is +/// a rename under a stable id, and the registered type names the id. +fn encode_store_snapshot(entries: &[CultCacheEnvelope], disk: &DiskStore) -> Result { + let arrived: BTreeMap<(String, String), &CultCacheEnvelope> = disk.envelopes.iter().map(|entry| (entry_id(entry), entry)).collect(); + // Per schema id and tier: the type its records carry, and the key of the record that named it. + let mut schema_types = BTreeMap::<(String, bool), (String, String)>::new(); for entry in entries { let schema_id = entry .schema_id .clone() .unwrap_or_else(|| entry.r#type.clone()); - if let Some(existing_type) = schema_types.insert(schema_id.clone(), entry.r#type.clone()) { - ensure!( - existing_type == entry.r#type, - "CultCache schema {schema_id:?} cannot identify both {existing_type:?} and {:?}", - entry.r#type - ); + let registered = arrived.get(&entry_id(entry)) != Some(&entry); + if let Some((existing_type, _)) = schema_types.insert((schema_id.clone(), registered), (entry.r#type.clone(), entry.key.clone())) { + if existing_type != entry.r#type { + return Err(anyhow::Error::new(SchemaConflictError { + schema_id, + schema_names: vec![existing_type, entry.r#type.clone()], + record_key: entry.key.clone(), + })); + } } } - let catalog = schema_types - .into_iter() - .map(|(schema_id, document_type)| { - PersistedSchemaCatalogEntry( - schema_id.clone(), - document_type, - format!("{schema_id}.v1"), - schema_id.clone(), - format!( - "{{\"schemaName\":\"{}\",\"schemaVersion\":\"{}.v1\",\"members\":[]}}", - escape_json_string(&schema_id), - escape_json_string(&schema_id) - ), - vec![schema_id], - Vec::new(), - ) - }) + let mut named = BTreeMap::::new(); + for ((schema_id, registered), (document_type, key)) in schema_types { + if !registered { + named.insert(schema_id, document_type); + } else if let Some(arrived_type) = named.insert(schema_id.clone(), document_type.clone()) + && arrived_type != document_type + && schema_id != document_type + { + return Err(anyhow::Error::new(SchemaConflictError { + schema_id, + schema_names: vec![arrived_type, document_type], + record_key: key, + })); + } + } + // An entry the file holds for an id is written back as the file holds it: this runtime keeps no description of another + // runtime's schema, so it never replaces one. It describes an id itself only when the file has no entry for it, or when a + // registered type owns the id under another name (a rename, where the registered type names the id). + let owned: BTreeSet<&str> = entries + .iter() + .filter(|entry| entry.schema_id.as_deref().is_none_or(|schema_id| schema_id == entry.r#type)) + .map(|entry| entry.r#type.as_str()) .collect(); + let mut catalog = BTreeMap::::new(); + for (schema_id, document_type) in named { + let held = disk + .catalog + .iter() + .find(|entry| entry.0 == schema_id) + .or_else(|| disk.catalog.iter().find(|entry| entry.5.contains(&schema_id))) + .filter(|entry| !owned.contains(schema_id.as_str()) || entry.1 == document_type); + let entry = match held { + Some(entry) => entry.clone(), + None => described(&schema_id, document_type), + }; + catalog.entry(entry.0.clone()).or_insert(entry); + } let records = entries .iter() .map(|entry| { @@ -3152,51 +3292,101 @@ fn encode_store_snapshot(entries: &[CultCacheEnvelope]) -> Result Result> { +/// The entry this runtime writes for a schema id it describes itself: its entry type is the id and the schema's name. +fn described(schema_id: &str, document_type: String) -> PersistedSchemaCatalogEntry { + PersistedSchemaCatalogEntry( + schema_id.to_string(), + document_type, + format!("{schema_id}.v1"), + schema_id.to_string(), + format!( + "{{\"schemaName\":\"{}\",\"schemaVersion\":\"{}.v1\",\"members\":[]}}", + escape_json_string(schema_id), + escape_json_string(schema_id) + ), + vec![schema_id.to_string()], + Vec::new(), + ) +} + +fn readable_store_format(header: &str) -> Option<&'static str> { + match header { + STORE_FORMAT_V1 => Some(STORE_FORMAT_V1), + STORE_FORMAT_ELEMENT_IDS => Some(STORE_FORMAT_ELEMENT_IDS), + _ => None, + } +} + +/// One store file's bytes, decoded: the header it carries (the one a rewrite keeps) and its envelopes. `header` is +/// [`store_header`]'s first slot and the bytes are already proven one complete array. An array whose first slot is not a +/// store header is the legacy envelope array. +fn decode_store_file(bytes: &[u8], header: Option<&str>) -> Result { + match header { + Some(header) => { + let format = readable_store_format(header) + .ok_or_else(|| anyhow!("the first slot is not a store header this runtime reads"))?; + let (envelopes, catalog) = decode_store_snapshot(bytes)?; + Ok(DiskStore { absent: false, format, envelopes, catalog }) + } + None => Ok(DiskStore { absent: false, format: STORE_FORMAT_V1, envelopes: decode_legacy_envelopes(bytes)?, catalog: Vec::new() }), + } +} + +/// What a store file holds, as its one reader decoded it: the header a rewrite keeps, its envelopes, and the catalog entries it +/// publishes them by, which a rewrite lays back for the ids they name. +struct DiskStore { + /// Nothing was at the path; the other fields describe an empty v1 store. + absent: bool, + format: &'static str, + envelopes: Vec, + catalog: Vec, +} + +impl DiskStore { + /// A file that is gone, or a dangling link a whole-file replace is about to replace: an empty v1 store. + fn absent() -> Self { + Self { absent: true, format: STORE_FORMAT_V1, envelopes: Vec::new(), catalog: Vec::new() } + } +} + +/// The legacy envelope array: an array of envelope maps. Its acceptance is the one verdict on an array whose first slot +/// is not a store header, for reading it and for replacing it. +fn decode_legacy_envelopes(bytes: &[u8]) -> Result> { + decode_grown(bytes).context("failed to decode legacy CultCache envelope array") +} + +/// Decodes a store whose header the caller has already read as a readable one: its envelopes and its catalog. +fn decode_store_snapshot(bytes: &[u8]) -> Result<(Vec, Vec)> { let snapshot: PersistedStoreSnapshot = decode_grown(bytes).context("failed to decode CultCache v1 snapshot")?; - let catalog = snapshot - .1 - .into_iter() - .map(|entry| (entry.0, entry.1)) - .collect::>(); - snapshot + // A schema id names the entry that has it as its own id; only an id no entry owns names the entry that lists it as compatible. + let mut catalog = BTreeMap::::new(); + for entry in &snapshot.1 { + catalog.entry(entry.0.clone()).or_insert_with(|| entry.1.clone()); + } + for entry in &snapshot.1 { + for compatible in &entry.5 { + catalog.entry(compatible.clone()).or_insert_with(|| entry.1.clone()); + } + } + let envelopes = snapshot .2 .into_iter() .map(|record| { let r#type = match catalog.get(&record.1) { Some(schema_name) => schema_name.clone(), - None => { - // A record can outlive its catalog entry when another runtime - // wrote it, or when the record is older than the catalog it - // was stamped against. The payload is a MessagePack array - // whose first field is the schema version, so the type can be - // recovered from the record itself rather than failing the - // whole snapshot. - let schema_version = - infer_schema_version_from_payload(&record.3).ok_or_else(|| { - anyhow!( - "CultCache record {:?} references missing schema {:?}", - record.0, - record.1 - ) - })?; - infer_schema_name(&schema_version).ok_or_else(|| { - anyhow!( - "CultCache record {:?} references missing schema {:?}", - record.0, - record.1 - ) - })? - } + None => bail!( + "CultCache record {:?} references schema {:?}, which the store's catalog does not publish", + record.0, + record.1 + ), }; Ok(CultCacheEnvelope { key: record.0, @@ -3206,17 +3396,25 @@ fn decode_store_snapshot(bytes: &[u8]) -> Result> { schema_id: Some(record.1), }) }) - .collect() + .collect::>>()?; + Ok((envelopes, snapshot.1)) } -/// A record recovered from its payload carries a schema name, while the -/// registry is keyed by entry type. Accept either, so a snapshot whose catalog -/// entry is gone still resolves to the registered definition. +/// A record resolves by its schema id: a registered type's id is its entry type. The name its catalog carries is metadata, +/// and names a type only when the id names none: the registry is keyed by entry type, and a store from a runtime with other +/// schema ids can be read by the type or schema name it carries. fn resolve_registered_type( known_types: &BTreeSet, schema_name_definitions: &BTreeMap, + schema_id: Option<&str>, persisted_type: &str, ) -> Option { + if let Some(schema_id) = schema_id + && known_types.contains(schema_id) + { + return Some(schema_id.to_string()); + } + if known_types.contains(persisted_type) { return Some(persisted_type.to_string()); } @@ -3224,68 +3422,6 @@ fn resolve_registered_type( schema_name_definitions.get(persisted_type).cloned() } -fn infer_schema_version_from_payload(payload: &[u8]) -> Option { - let mut offset = 0usize; - read_array_header(payload, &mut offset)?; - read_string(payload, &mut offset) -} - -fn read_array_header(payload: &[u8], offset: &mut usize) -> Option { - let marker = *payload.get(*offset)?; - *offset += 1; - match marker { - 0x90..=0x9f => Some((marker & 0x0f) as u32), - 0xdc => { - let bytes = payload.get(*offset..(*offset + 2))?; - *offset += 2; - Some(u16::from_be_bytes([bytes[0], bytes[1]]) as u32) - } - 0xdd => { - let bytes = payload.get(*offset..(*offset + 4))?; - *offset += 4; - Some(u32::from_be_bytes([bytes[0], bytes[1], bytes[2], bytes[3]])) - } - _ => None, - } -} - -fn read_string(payload: &[u8], offset: &mut usize) -> Option { - let marker = *payload.get(*offset)?; - *offset += 1; - let length = match marker { - 0xa0..=0xbf => (marker & 0x1f) as usize, - 0xd9 => { - let length = *payload.get(*offset)? as usize; - *offset += 1; - length - } - 0xda => { - let bytes = payload.get(*offset..(*offset + 2))?; - *offset += 2; - u16::from_be_bytes([bytes[0], bytes[1]]) as usize - } - 0xdb => { - let bytes = payload.get(*offset..(*offset + 4))?; - *offset += 4; - u32::from_be_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]) as usize - } - _ => return None, - }; - - let bytes = payload.get(*offset..(*offset + length))?; - std::str::from_utf8(bytes).ok().map(str::to_string) -} - -fn infer_schema_name(schema_version: &str) -> Option { - let marker = schema_version.rfind(".v")?; - let version = schema_version.get((marker + 2)..)?; - if marker == 0 || version.is_empty() || !version.bytes().all(|value| value.is_ascii_digit()) { - return None; - } - - Some(schema_version[..marker].to_string()) -} - fn escape_json_string(value: &str) -> String { value.replace('\\', "\\\\").replace('"', "\\\"") } @@ -3329,6 +3465,12 @@ fn release_lock(lock: File, result: Result) -> Result { result } +fn io_not_found(error: &anyhow::Error) -> bool { + error + .chain() + .any(|cause| cause.downcast_ref::().is_some_and(|io| io.kind() == std::io::ErrorKind::NotFound)) +} + /// True only when nothing is at `path`; a symbolic link, dangling or not, is something. /// A path that cannot be inspected (a file where a directory should be, no permission) /// is an error, never absent. @@ -3379,7 +3521,7 @@ fn remove_abandoned_staging_files(destination: &Path) -> Result<()> { #[cfg(unix)] fn replace_file_atomically(staged: &Path, destination: &Path) -> Result<()> { - fs::rename(staged, destination).with_context(|| { + injected_write_fault(WriteStep::Rename).and_then(|()| fs::rename(staged, destination)).with_context(|| { format!( "failed to atomically replace {} with {}", destination.display(), @@ -3395,6 +3537,8 @@ fn replace_file_atomically(staged: &Path, destination: &Path) -> Result<()> { MOVEFILE_REPLACE_EXISTING, MOVEFILE_WRITE_THROUGH, MoveFileExW, }; + injected_write_fault(WriteStep::Rename).context("failed to atomically replace the store")?; + let staged_wide = staged .as_os_str() .encode_wide() @@ -3495,6 +3639,7 @@ fn sync_parent_directory(path: &Path) -> Result<()> { #[allow(dead_code)] enum WriteStep { StagingSync, + Rename, DirectorySync, Unlock, } @@ -5033,7 +5178,7 @@ mod tests { } #[test] - fn messagepack_store_recovers_schema_stamped_records_missing_catalog_entries() -> Result<()> { + fn messagepack_store_refuses_a_record_whose_schema_the_catalog_does_not_publish() -> Result<()> { let temp = tempfile::tempdir()?; let store_path = temp.path().join("missing-catalog.msgpack"); let expected = SchemaStamped { @@ -5041,8 +5186,7 @@ mod tests { name: "schema-stamped".to_string(), value: "still readable".to_string(), }; - // An empty catalog with a stale schema id is what a record written by - // another runtime, or one older than its catalog, looks like on disk. + // An empty catalog with a stale schema id. let snapshot = PersistedStoreSnapshot( "cultcache.store.v1".to_string(), Vec::new(), @@ -5058,15 +5202,9 @@ mod tests { let mut cache = CultCache::new(); cache.register_entry_type::()?; cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path))?; - cache.pull_all_backing_stores()?; - - assert_eq!(cache.get_required::("record-1")?, expected); - assert_eq!( - cache - .get_required_envelope::("record-1")? - .schema_id, - Some("sha256:stale-schema-id-from-cold-record".to_string()) - ); + // The payload opens with a schema version, and the catalog is the store's own description: no recovery. + let error = cache.pull_all_backing_stores().unwrap_err(); + assert!(error.downcast_ref::().is_some(), "{error:#}"); Ok(()) } @@ -5350,6 +5488,243 @@ mod tests { Ok(()) } + // The element-id marker: tests/vectors/document-variants-c2a/v3-base.msgpack is v1-base with its header replaced. + fn header_after_rewrite(vector: &str) -> Result { + let temp = tempfile::tempdir()?; + let path = temp.path().join("store.msgpack"); + let source = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../tests/vectors") + .join(vector); + std::fs::copy(source, &path)?; + let mut store = SingleFileMessagePackBackingStore::new(&path); + let envelopes = store.pull_all()?; + let keys: Vec<_> = envelopes.iter().map(|e| e.key.as_str()).collect(); + assert_eq!(keys, vec!["alpha", "beta"]); + store.push(&envelopes[0])?; + Ok(store_header(&std::fs::read(&path)?).first.expect("a store header")) + } + + #[test] + fn a_v3_element_id_store_reads_and_a_rewrite_keeps_the_marker() -> Result<()> { + assert_eq!( + header_after_rewrite("document-variants-c2a/v3-base.msgpack")?, + "cultcache.store.v3" + ); + Ok(()) + } + + // A whole-store flush reads the header the file carries first: one it cannot read refuses, and the file is untouched. + #[test] + fn push_all_refuses_a_store_whose_header_it_cannot_read_and_leaves_it_untouched() -> Result<()> { + for vector in [ + "document-variants-c0/unknown-header.msgpack", + "document-variants-c0/variant-v2.msgpack", + "document-variants-c1/variant-store.msgpack", + ] { + let temp = tempfile::tempdir()?; + let path = temp.path().join("store.msgpack"); + let source = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../tests/vectors").join(vector); + std::fs::copy(source, &path)?; + let before = std::fs::read(&path)?; + let mut store = SingleFileMessagePackBackingStore::new(&path); + let error = store.push_all(&[snapshot_envelope("x", b"one")], PushAllOptions::default()).unwrap_err(); + assert_eq!(unreadable_kind(&error), Some(CultCacheStoreUnreadableKind::UnsupportedFormat), "{vector}: {error:#}"); + assert_eq!(std::fs::read(&path)?, before, "{vector} was rewritten"); + } + Ok(()) + } + + // The header is read by the one snapshot reader, whatever string encoding carries it. + #[test] + fn push_all_refuses_a_header_it_cannot_read_however_the_string_is_encoded() -> Result<()> { + let long = format!("cultcache.store.v9.{}", "x".repeat(90)); + let mut str16 = vec![0x93, 0xda, 0x00, 18]; + str16.extend_from_slice(b"cultcache.store.v9"); + str16.extend_from_slice(&[0x90, 0x90]); + let mut str8 = vec![0x93, 0xd9, long.len() as u8]; + str8.extend_from_slice(long.as_bytes()); + str8.extend_from_slice(&[0x90, 0x90]); + for (name, bytes) in [("str16", str16), ("str8 over 90 bytes", str8)] { + let temp = tempfile::tempdir()?; + let path = temp.path().join("store.msgpack"); + std::fs::write(&path, &bytes)?; + let mut store = SingleFileMessagePackBackingStore::new(&path); + let error = store.push_all(&[snapshot_envelope("x", b"one")], PushAllOptions::default()).unwrap_err(); + assert_eq!(unreadable_kind(&error), Some(CultCacheStoreUnreadableKind::UnsupportedFormat), "{name}: {error:#}"); + assert_eq!(std::fs::read(&path)?, bytes, "{name} was rewritten"); + } + Ok(()) + } + + + // A file is replaced by a rewrite exactly when this runtime's own reader opens it: one verdict per file, asked by open, + // push_all and push alike. The bytes and every runtime's verdict are tests/vectors/document-variants-c2a/readability. + // A refusal is a CultCacheStoreUnreadable, and a replaced file carries the header its old content decides. + #[test] + fn a_file_is_replaced_exactly_when_it_reads() -> Result<()> { + let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../tests/vectors/document-variants-c2a/readability"); + let manifest = std::fs::read_to_string(root.join("manifest.txt"))?; + let mut listed = Vec::new(); + for line in manifest.lines().filter(|line| !line.is_empty() && !line.starts_with('#')) { + let cells: Vec<&str> = line.split_whitespace().collect(); + let (vector, reads) = (cells[0], cells[2] == "reads"); + if !vector.starts_with("..") { + listed.push(vector.to_string()); + } + let bytes = std::fs::read(root.join(vector))?; + // A v3 store keeps its marker through a rewrite; every other file that reads is written v1. + let header = if vector.contains("v3") { "cultcache.store.v3" } else { STORE_FORMAT_V1 }; + for operation in ["open", "soft_push_all", "push_all", "push"] { + let temp = tempfile::tempdir()?; + let path = temp.path().join("store.msgpack"); + std::fs::write(&path, &bytes)?; + let mut store = SingleFileMessagePackBackingStore::new(&path); + let outcome = match operation { + "open" => store.pull_all().map(|_| ()), + "soft_push_all" => store.push_all(&[snapshot_envelope("x", b"one")], PushAllOptions { soft: true }), + "push_all" => store.push_all(&[snapshot_envelope("x", b"one")], PushAllOptions::default()), + _ => store.push(&snapshot_envelope("x", b"one")), + }; + let what = format!("{vector} {operation}"); + assert_eq!(outcome.is_ok(), reads, "{what}: {outcome:?}"); + if operation == "soft_push_all" && reads { + // A soft flush leaves a store it reads byte for byte. + assert_eq!(std::fs::read(&path)?, bytes, "{what} rewrote a store it reads"); + continue; + } + if !reads { + let error = outcome.unwrap_err(); + let expected = if vector == "header-v9.bin" { + CultCacheStoreUnreadableKind::UnsupportedFormat + } else { + CultCacheStoreUnreadableKind::Undecodable + }; + assert_eq!(unreadable_kind(&error), Some(expected), "{what}: {error:#}"); + assert!(io_error_in(&error).is_none(), "{what}: a refusal is not an I/O failure: {error:#}"); + assert_eq!(std::fs::read(&path)?, bytes, "{what} rewrote a file it cannot read"); + } else if operation != "open" { + assert_eq!(store_header(&std::fs::read(&path)?).first.as_deref(), Some(header), "{what}"); + assert!(store.pull_all()?.iter().any(|entry| entry.key == "x"), "{what}"); + } + } + } + // Every vector in the folder has a manifest row. + let mut files: Vec = std::fs::read_dir(&root)? + .filter_map(|entry| entry.ok()?.file_name().into_string().ok()) + .filter(|name| name.ends_with(".bin")) + .collect(); + files.sort(); + listed.sort(); + assert_eq!(listed, files); + Ok(()) + } + + // An id one entry owns and a later entry lists as compatible names the entry that owns it. + #[test] + fn an_id_an_entry_owns_names_that_entry_not_one_that_lists_it_as_compatible() -> Result<()> { + for vector in ["own-id-over-compatible-v3.bin", "compatible-before-owner-v3.bin"] { + let temp = tempfile::tempdir()?; + let path = temp.path().join("store.msgpack"); + std::fs::copy( + std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../tests/vectors/document-variants-c2a/readability") + .join(vector), + &path, + )?; + let envelopes = SingleFileMessagePackBackingStore::new(&path).pull_all()?; + assert_eq!(envelopes.len(), 2, "{vector}"); + assert!(envelopes.iter().all(|envelope| envelope.r#type == "vectors.item"), "{vector}: {envelopes:?}"); + } + Ok(()) + } + + // Records of different types under one schema id cannot be described by one catalog entry: the write is refused, typed, in + // either order, and the file is left as it was. A rename with a stable id (same type name in the new schema) writes fine. + #[test] + fn records_of_different_types_under_one_schema_id_refuse_the_write_typed() -> Result<()> { + let record = |key: &str, r#type: &str| CultCacheEnvelope { + key: key.into(), + r#type: r#type.into(), + payload: b"one".to_vec(), + stored_at: "2026-09-30T00:00:00Z".into(), + schema_id: Some("tests.x".into()), + }; + for order in [vec![record("a", "tests.n"), record("b", "tests.m")], vec![record("b", "tests.m"), record("a", "tests.n")]] { + let temp = tempfile::tempdir()?; + let path = temp.path().join("store.msgpack"); + let mut store = SingleFileMessagePackBackingStore::new(&path); + let error = store.push_all(&order, PushAllOptions::default()).unwrap_err(); + let conflict = error.downcast_ref::().unwrap_or_else(|| panic!("{error:#}")); + assert_eq!(conflict.schema_id, "tests.x"); + let mut names = conflict.schema_names.clone(); + names.sort(); + assert_eq!(names, vec!["tests.m".to_string(), "tests.n".to_string()]); + assert!(conflict.record_key == "a" || conflict.record_key == "b"); + assert!(!path.exists(), "nothing is written"); + } + Ok(()) + } + + // Another runtime's catalog entry survives this runtime's writes: a write of any record lays back, byte for byte, the entry the + // file holds for an id this runtime does not own. This one carries what a C# writer publishes (a version, a content hash, + // canonical JSON, members), none of which this runtime could describe. Whole-store writes and single pushes alike. + #[test] + fn a_write_lays_back_the_catalog_entry_the_file_holds_for_an_id_it_does_not_own() -> Result<()> { + let held = PersistedSchemaCatalogEntry( + "sha256:csharp".into(), + "tests.note".into(), + "tests.note_v7".into(), + "sha256:content".into(), + "{\"schemaName\":\"tests.note\"}".into(), + vec!["sha256:csharp".into()], + vec![PersistedSchemaCatalogMember(0, "Title".into(), "System.String".into(), false, false, None, true, Some("title".into()))], + ); + let own = CultCacheEnvelope { + key: "b".into(), + r#type: "tests.other".into(), + payload: b"two".to_vec(), + stored_at: "2026-09-30T00:00:00Z".into(), + schema_id: Some("tests.other".into()), + }; + // An older entry that lists the id: the entry that owns the id is the one laid back. + let lister = PersistedSchemaCatalogEntry( + "sha256:older".into(), + "tests.note".into(), + "tests.note_v6".into(), + "sha256:older".into(), + String::new(), + vec!["sha256:older".into(), "sha256:csharp".into()], + Vec::new(), + ); + for whole_store in [false, true] { + let temp = tempfile::tempdir()?; + let path = temp.path().join("store.msgpack"); + let record = PersistedRecord("a".into(), "sha256:csharp".into(), "2026-09-30T00:00:00Z".into(), b"one".to_vec()); + std::fs::write( + &path, + rmp_serde::to_vec(&PersistedStoreSnapshot("cultcache.store.v1".into(), vec![lister.clone(), held.clone()], vec![record]))?, + )?; + let mut store = SingleFileMessagePackBackingStore::new(&path); + if whole_store { + // As a Rust cache holds it after a pull: under the registered type that reads the schema by name. + let mut entries = store.pull_all()?; + for entry in &mut entries { + entry.r#type = "tests.rust_note".into(); + } + entries.push(own.clone()); + store.push_all(&entries, PushAllOptions::default())?; + } else { + store.push(&own)?; + } + + let written: PersistedStoreSnapshot = rmp_serde::from_slice(&std::fs::read(&path)?)?; + let entry = written.1.iter().find(|entry| entry.0 == "sha256:csharp").expect("the entry for the id is written"); + assert_eq!(rmp_serde::to_vec(entry)?, rmp_serde::to_vec(&held)?, "whole store: {whole_store}"); + assert!(written.1.iter().any(|entry| entry.0 == "tests.other"), "its own id is described"); + } + Ok(()) + } + // Store read and write faults: what a consumer can tell apart, and what a failed // write leaves on disk. @@ -5454,41 +5829,6 @@ mod tests { Ok(()) } - #[test] - fn push_all_refuses_a_store_in_a_format_this_runtime_does_not_read_and_leaves_it() -> Result<()> { - let vector = fs::read( - Path::new(env!("CARGO_MANIFEST_DIR")).join("../../tests/vectors/document-variants-c2a/v3-base.msgpack"), - )?; - let entry = CultCacheEnvelope { - key: "probe".into(), - r#type: "probe".into(), - payload: vec![0x80], - stored_at: "2026-10-22T00:00:00Z".into(), - schema_id: None, - }; - // A newer store whole, and cut short after its header: the header alone is judged. - for bytes in [&vector[..], &vector[..vector.len() / 2]] { - let temp = tempfile::tempdir()?; - let path = temp.path().join("store.msgpack"); - fs::write(&path, bytes)?; - let error = SingleFileMessagePackBackingStore::new(&path) - .push_all(std::slice::from_ref(&entry), PushAllOptions::default()) - .unwrap_err(); - assert_eq!(unreadable_kind(&error), Some(CultCacheStoreUnreadableKind::UnsupportedFormat), "{error:#}"); - assert!(format!("{error:#}").contains("cultcache.store.v3"), "{error:#}"); - assert_eq!(fs::read(&path)?, bytes); - } - // A file with no readable stamp is not a newer store: a whole write still heals it. - let temp = tempfile::tempdir()?; - let path = temp.path().join("store.msgpack"); - fs::write(&path, [0x93u8, 0xa0])?; - let mut store = SingleFileMessagePackBackingStore::new(&path); - store.push_all(std::slice::from_ref(&entry), PushAllOptions::default())?; - assert_eq!(store.pull_all()?.len(), 1); - Ok(()) - } - - #[test] fn a_store_that_cannot_be_read_is_an_io_error_and_never_unreadable() -> Result<()> { let temp = tempfile::tempdir()?; @@ -5658,6 +5998,141 @@ mod tests { Ok(()) } + fn store_file(path: &std::path::Path, catalog: &[(&str, &str)], records: &[(&str, &str, Vec)]) -> Result<()> { + let snapshot = PersistedStoreSnapshot( + "cultcache.store.v1".to_string(), + catalog + .iter() + .map(|(id, name)| { + PersistedSchemaCatalogEntry( + id.to_string(), + name.to_string(), + format!("{name}.v1"), + id.to_string(), + String::new(), + vec![id.to_string()], + Vec::new(), + ) + }) + .collect(), + records + .iter() + .map(|(key, id, payload)| { + PersistedRecord(key.to_string(), id.to_string(), "2026-09-30T00:00:00Z".to_string(), payload.clone()) + }) + .collect(), + ); + std::fs::write(path, rmp_serde::to_vec(&snapshot)?)?; + Ok(()) + } + + // A schema renamed under a stable id: a registered type owns its id (its id is its entry type), so a write of it onto a file + // whose entry carries the old name is not a conflict. Every record under the id, the file's and the write's, is then described + // by the registered name. A write of another type under that id retypes what the file holds, and is refused. + #[test] + fn a_schema_renamed_under_a_stable_id_is_written_under_the_registered_name() -> Result<()> { + let envelope = |r#type: &str, schema_id: &str| CultCacheEnvelope { + key: "b".into(), + r#type: r#type.into(), + payload: b"two".to_vec(), + stored_at: "2026-09-30T00:00:00Z".into(), + schema_id: Some(schema_id.into()), + }; + let temp = tempfile::tempdir()?; + let path = temp.path().join("store.msgpack"); + store_file(&path, &[("t.s", "t.old")], &[("a", "t.s", b"one".to_vec())])?; + SingleFileMessagePackBackingStore::new(&path).push(&envelope("t.s", "t.s"))?; + + let mut envelopes = SingleFileMessagePackBackingStore::new(&path).pull_all()?; + envelopes.sort_by(|left, right| left.key.cmp(&right.key)); + assert_eq!( + envelopes.iter().map(|entry| (entry.key.as_str(), entry.r#type.as_str())).collect::>(), + vec![("a", "t.s"), ("b", "t.s")] + ); + + store_file(&path, &[("t.s", "t.old")], &[("a", "t.s", b"one".to_vec())])?; + let before = std::fs::read(&path)?; + let error = SingleFileMessagePackBackingStore::new(&path).push(&envelope("t.new", "t.s")).unwrap_err(); + let conflict = error.downcast_ref::().unwrap_or_else(|| panic!("{error:#}")); + assert_eq!(conflict.schema_id, "t.s"); + assert_eq!(conflict.record_key, "b"); + assert_eq!(std::fs::read(&path)?, before, "the file is left as it was"); + Ok(()) + } + + #[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] + #[cultcache(type = "tests.stable", schema = "tests.old_name")] + struct StableId { + #[cultcache(key = 0)] + name: String, + } + + #[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] + #[cultcache(type = "tests.other", schema = "tests.other_name")] + struct OtherId { + #[cultcache(key = 0)] + name: String, + } + + // A record resolves to a registered type by its schema id, never by the name its catalog entry carries: a schema renamed under a + // stable id opens, and an id that names one type is not overruled by a name that names another. + #[test] + fn a_schema_renamed_under_a_stable_id_opens_by_its_id() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = temp.path().join("store.msgpack"); + store_file(&path, &[("tests.stable", "tests.renamed")], &[("a", "tests.stable", rmp_serde::to_vec(&StableId { name: "a".into() })?)])?; + let mut cache = CultCache::new(); + cache.register_entry_type::()?; + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&path))?; + cache.pull_all_backing_stores()?; + assert_eq!(cache.get_required::("a")?.name, "a"); + Ok(()) + } + + #[test] + fn a_schema_id_that_names_one_type_is_not_overruled_by_a_name_that_names_another() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = temp.path().join("store.msgpack"); + // The entry that publishes the id carries the other type's name. + store_file(&path, &[("tests.other", "tests.stable")], &[("k", "tests.other", rmp_serde::to_vec(&OtherId { name: "k".into() })?)])?; + let mut cache = CultCache::new(); + cache.register_entry_type::()?; + cache.register_entry_type::()?; + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&path))?; + cache.pull_all_backing_stores()?; + assert_eq!(cache.get_required::("k")?.name, "k"); + assert!(cache.get::("k")?.is_none()); + Ok(()) + } + + // A store cut short is refused whatever its version, and left as it is. + #[test] + fn a_truncated_v1_or_v2_store_is_refused_on_open_and_rewrite() -> Result<()> { + for vector in [ + "document-variants-c0/v1-base.msgpack", + "document-variants-c0/variant-v2.msgpack", + "document-variants-c1/variant-store.msgpack", + ] { + let source = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../tests/vectors").join(vector); + let whole = std::fs::read(source)?; + let cut = &whole[..whole.len() - 1]; + for operation in ["open", "push_all"] { + let temp = tempfile::tempdir()?; + let path = temp.path().join("store.msgpack"); + std::fs::write(&path, cut)?; + let mut store = SingleFileMessagePackBackingStore::new(&path); + let outcome = match operation { + "open" => store.pull_all().map(|_| ()), + _ => store.push_all(&[snapshot_envelope("x", b"one")], PushAllOptions::default()), + }; + let error = outcome.expect_err(&format!("{vector} {operation}")); + assert!(unreadable_kind(&error).is_some(), "{vector} {operation}: {error:#}"); + assert_eq!(std::fs::read(&path)?, cut, "{vector} {operation} rewrote a truncated store"); + } + } + Ok(()) + } + #[test] fn refusals_say_what_was_found_without_blaming_variants() { let unsupported = format!("{:#}", pull_vector("unknown-header.msgpack").unwrap_err()); @@ -5673,11 +6148,10 @@ mod tests { assert!(!invalid.contains("runtime that"), "{invalid}"); } - /// The store path is a directory, so the staging file is written and synced and - /// then cannot be renamed over it: a failure after the write, reached without - /// permissions. + /// The store path is a directory, so the one reader cannot reach a store there: the write stops at the read, as an + /// I/O error and not a write failure, before any staging file exists. #[test] - fn a_write_that_fails_at_the_rename_leaves_no_staging_file() -> Result<()> { + fn a_write_over_a_directory_stops_at_the_read_and_leaves_no_staging_file() -> Result<()> { let temp = tempfile::tempdir()?; let path = temp.path().join("store.cc"); fs::create_dir(&path)?; @@ -5687,7 +6161,7 @@ mod tests { let error = store .push_all(&[snapshot_envelope("alpha", b"1")], PushAllOptions::default()) .unwrap_err(); - assert!(format!("{error:#}").contains("failed to atomically replace"), "{error:#}"); + assert!(io_error_in(&error).is_some() && write_failure_kind(&error).is_none(), "{error:#}"); let staging: Vec<_> = fs::read_dir(temp.path())? .map(|entry| entry.map(|entry| entry.file_name().to_string_lossy().into_owned())) @@ -5736,6 +6210,15 @@ mod tests { Ok(()) } + #[test] + fn a_rewrite_of_a_v1_store_stays_v1() -> Result<()> { + assert_eq!( + header_after_rewrite("document-variants-c0/v1-base.msgpack")?, + "cultcache.store.v1" + ); + Ok(()) + } + #[test] fn a_header_is_echoed_only_in_the_known_shape() -> Result<()> { let temp = tempfile::tempdir()?; @@ -5746,6 +6229,8 @@ mod tests { ("cultcache.store.SECRET-HEADER", false), ("cultcache.store.v12SECRET", false), ("cultcache.store.v", false), + // Digits are ASCII only: Arabic-Indic digits are not the known shape. + ("cultcache.store.v١٢", false), ] { let mut bytes = vec![0x93]; bytes.extend(fixstr(header)); @@ -5754,6 +6239,9 @@ mod tests { let error = store.pull_all().unwrap_err(); let text = format!("{error:#}"); assert_eq!(unreadable_kind(&error), Some(CultCacheStoreUnreadableKind::UnsupportedFormat), "{text}"); + // The refusal names every format this runtime reads. + assert!(text.contains("\"cultcache.store.v1\""), "{text}"); + assert!(text.contains("\"cultcache.store.v3\""), "{text}"); if echoed { assert!(text.contains(&format!("\"{header}\"")), "{text}"); } else { @@ -5782,6 +6270,21 @@ mod tests { assert!(io_error_in(&error).is_some(), "{error:#}"); assert_eq!(unreadable_kind(&error), None, "{error:#}"); } + // Every merging writer refuses on its own read, so none can replace the link. + let mut writer = SingleFileMessagePackBackingStore::new(&path); + let delta = snapshot_envelope("delta", b"4"); + let writes = [ + writer.push(&delta), + writer.delete(&delta), + writer.insert_entry_if_absent(delta.clone()).map(|_| ()), + writer.compare_and_swap_batch(&[], vec![delta.clone()]).map(|_| ()), + writer.append_if_snapshot_unchanged(&[], vec![delta.clone()]).map(|_| ()), + ]; + for (index, result) in writes.into_iter().enumerate() { + let error = result.expect_err("a merging write through a dangling link must refuse"); + assert!(io_error_in(&error).is_some(), "write {index}: {error:#}"); + assert_eq!(write_failure_kind(&error), None, "write {index}: {error:#}"); + } assert!(fs::symlink_metadata(&path)?.file_type().is_symlink()); assert!(!target.exists()); assert_eq!(fs::read_dir(&volume)?.count(), 0); @@ -5817,18 +6320,46 @@ mod tests { assert_eq!(fs::read(&path)?, before); assert_eq!(fs::read_dir(temp.path())?.count(), 2, "only the store and its lock remain"); - // The rename fails when the store path is a directory. + // A store path that is a directory is refused at the read: an I/O error, not a write failure. let blocked = temp.path().join("blocked.cc"); fs::create_dir(&blocked)?; let mut blocked_store = SingleFileMessagePackBackingStore::new(&blocked); let error = blocked_store .push_all(&[snapshot_envelope("alpha", b"1")], PushAllOptions::default()) .unwrap_err(); - assert_eq!(write_failure_kind(&error), Some(CultCacheStoreWriteFailedKind::NotReplaced), "{error:#}"); + assert!(io_error_in(&error).is_some() && write_failure_kind(&error).is_none(), "{error:#}"); assert!(fs::metadata(&blocked)?.is_dir()); Ok(()) } + #[test] + fn a_write_whose_rename_fails_is_not_replaced_and_leaves_no_staging_file() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = temp.path().join("store.cc"); + let (mut store, before) = written_store(&path)?; + let replacement = [snapshot_envelope("delta", b"4")]; + + let error = with_write_fault(WriteStep::Rename, || { + store.compare_exchange_snapshot(&store.pull_all().unwrap(), &replacement) + }) + .expect_err("a rename that failed is not a write that landed"); + assert_eq!(write_failure_kind(&error), Some(CultCacheStoreWriteFailedKind::NotReplaced), "{error:#}"); + assert!(io_error_in(&error).is_some(), "{error:#}"); + assert_eq!(fs::read(&path)?, before, "the store is as it was"); + assert_eq!(fs::read_dir(temp.path())?.count(), 2, "only the store and its lock remain: no staging file"); + + let error = with_write_fault(WriteStep::Rename, || store.push(&replacement[0])) + .expect_err("a push whose rename failed is not a push that landed"); + assert_eq!(write_failure_kind(&error), Some(CultCacheStoreWriteFailedKind::NotReplaced), "{error:#}"); + assert_eq!(fs::read(&path)?, before); + assert_eq!(fs::read_dir(temp.path())?.count(), 2, "only the store and its lock remain: no staging file"); + + // The failure left nothing behind that stops the next write. + store.push(&replacement[0])?; + assert!(store.pull_all()?.contains(&replacement[0])); + Ok(()) + } + #[cfg(unix)] #[test] fn a_write_whose_directory_sync_fails_is_replaced_but_not_durable() -> Result<()> { @@ -5911,6 +6442,34 @@ mod tests { Ok(()) } + // The two cases the vectors cannot hold: nothing at the path is written, and a dangling link is refused and left as it is. + #[test] + fn a_soft_push_all_writes_where_nothing_is_and_refuses_a_dangling_link() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = temp.path().join("store.cc"); + SingleFileMessagePackBackingStore::new(&path).push_all(&[snapshot_envelope("a", b"1")], PushAllOptions { soft: true })?; + let seeded = fs::read(&path)?; + let mut store = SingleFileMessagePackBackingStore::new(&path); + assert_eq!(store.pull_all()?.len(), 1); + store.push_all(&[snapshot_envelope("b", b"2")], PushAllOptions { soft: true })?; + assert_eq!(fs::read(&path)?, seeded, "a soft flush over a store it reads wrote"); + + #[cfg(unix)] + { + let volume = temp.path().join("volume"); + fs::create_dir(&volume)?; + let link = temp.path().join("dangling.cc"); + std::os::unix::fs::symlink(volume.join("store.cc"), &link)?; + let error = SingleFileMessagePackBackingStore::new(&link) + .push_all(&[snapshot_envelope("a", b"1")], PushAllOptions { soft: true }) + .unwrap_err(); + assert!(io_not_found(&error), "{error:#}"); + assert!(fs::symlink_metadata(&link)?.file_type().is_symlink(), "the link was replaced"); + assert_eq!(fs::read_dir(&volume)?.count(), 0, "nothing reached the link's volume"); + } + Ok(()) + } + #[cfg(unix)] #[test] fn r3_decides_resolve_or_refuse_a_push_all_through_a_dangling_link_replaces_the_link() -> Result<()> { @@ -6037,4 +6596,4 @@ mod tests { assert_eq!(fs::read(&path)?, before); Ok(()) } -} +} \ No newline at end of file diff --git a/packages/cultcache-rs/tests/union_parity.rs b/packages/cultcache-rs/tests/union_parity.rs index 6b120791..24fe999a 100644 --- a/packages/cultcache-rs/tests/union_parity.rs +++ b/packages/cultcache-rs/tests/union_parity.rs @@ -32,6 +32,8 @@ enum Node { struct WrapArm { #[cultcache(key = 0)] inner: Option>, + #[cultcache(key = 1)] + id: String, } #[derive(Clone, Debug, PartialEq, CultSlots)] @@ -85,6 +87,17 @@ enum Outcome { Pending(PendingArm), #[cultcache(key = 1)] Completed(CompletedArm), + #[cultcache(key = 2)] + Denied(IdArm), + #[cultcache(key = -2147483648)] + Floor(IdArm), + #[cultcache(key = 2147483647)] + Ceiling(IdArm), +} + +// The codec's unit arm (no members, written [key, []]), kept apart from the golden: a document list's arms carry ids. +#[derive(Clone, Debug, PartialEq, CultUnion)] +enum UnitOutcome { #[cultcache(key = 2)] Denied, #[cultcache(key = -2147483648)] @@ -97,12 +110,23 @@ enum Outcome { struct PendingArm { #[cultcache(key = 0)] message: String, + #[cultcache(key = 1)] + id: String, +} + +// An arm with no members of its own: just its element id. +#[derive(Clone, Debug, PartialEq, CultSlots)] +struct IdArm { + #[cultcache(key = 0)] + id: String, } #[derive(Clone, Debug, PartialEq, CultSlots)] struct CompletedArm { #[cultcache(key = 0)] source_version: i32, + #[cultcache(key = 1)] + id: String, } #[derive(Clone, Debug, PartialEq, CultSlots)] @@ -178,26 +202,26 @@ fn boxed(id: &str, children: Vec) -> Node { } fn constructed() -> UnionSurface { - let inner = boxed("inner", vec![text("deep", "x"), image("img", "uri://a", "alt")]); + let inner = boxed("0000000000d0", vec![text("0000000000d1", "x"), image("0000000000d2", "uri://a", "alt")]); UnionSurface { id: "surface".into(), version: 3, root: Some(boxed( - "root", + "0000000000c0", vec![ - text("t1", "hello"), + text("0000000000c1", "hello"), Node::Use(UseArm { - id: "u1".into(), + id: "0000000000c2".into(), composition: Composition { name: "comp".into(), body: Some(std::boxed::Box::new(inner)) }, - slots: vec![Some(text("s1", "slot")), None, Some(boxed("b2", vec![]))], + slots: vec![Some(text("0000000000e1", "slot")), None, Some(boxed("0000000000e2", vec![]))], }), - image("i2", "uri://b", ""), + image("0000000000c3", "uri://b", ""), ], )), outcomes: vec![ - Outcome::Pending(PendingArm { message: "wait".into() }), - Outcome::Completed(CompletedArm { source_version: 7 }), - Outcome::Denied, + Outcome::Pending(PendingArm { message: "wait".into(), id: "0000000000a1".into() }), + Outcome::Completed(CompletedArm { source_version: 7, id: "0000000000a2".into() }), + Outcome::Denied(id_arm("0000000000a3")), ], part: Some(Item::Tool(UsesArm { name: "hammer".into(), uses: 12 })), crafted: Some(CraftedItem::Part(CountArm { name: "gear".into(), count: 4 })), @@ -205,6 +229,10 @@ fn constructed() -> UnionSurface { } } +fn id_arm(id: &str) -> IdArm { + IdArm { id: id.into() } +} + fn vector(name: &str) -> Vec { let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("../../tests/vectors/union-parity") @@ -226,8 +254,25 @@ fn union_surface_decodes_and_reencodes() { #[test] fn a_unit_arm_is_a_key_and_an_empty_slots_array() { - assert_eq!(rmp_serde::to_vec(&Outcome::Denied).unwrap(), [0x92, 0x02, 0x90]); - assert_eq!(rmp_serde::from_slice::(&[0x92, 0x02, 0x90]).unwrap(), Outcome::Denied); + // The bytes are the C# reference's, not spelled here. + for (file, arm) in [ + ("unit-arm-denied.msgpack", UnitOutcome::Denied), + ("unit-arm-floor.msgpack", UnitOutcome::Floor), + ("unit-arm-ceiling.msgpack", UnitOutcome::Ceiling), + ] { + let reference = vector(file); + assert_eq!(rmp_serde::to_vec(&arm).unwrap(), reference, "{file}"); + assert_eq!(rmp_serde::from_slice::(&reference).unwrap(), arm, "{file}"); + } +} + +#[test] +fn an_id_only_arm_is_a_key_and_a_one_slot_array() { + let mut bytes = vec![0x92, 0x02, 0x91, 0xac]; + bytes.extend_from_slice(b"0000000000a3"); + let denied = Outcome::Denied(id_arm("0000000000a3")); + assert_eq!(rmp_serde::to_vec(&denied).unwrap(), bytes); + assert_eq!(rmp_serde::from_slice::(&bytes).unwrap(), denied); } #[test] @@ -246,7 +291,7 @@ fn union_refusal_vectors() { let key = key.expect(&file); assert!(message.contains(&format!("unknown arm key {key} ")), "{file}: {message}"); } - for canary in ["root", "hello", "uri://", "surface"] { + for canary in ["0000000000c0", "hello", "uri://", "surface"] { assert!(!message.contains(canary), "{file} echoed a value: {message}"); } } @@ -287,8 +332,8 @@ fn a_union_that_is_not_a_key_and_slots_pair_is_refused_naming_the_enum() { ("a unit arm with slots", rmp_serde::to_vec(&(2u64, ("CANARY-slot",))).unwrap()), ]; for (label, bytes) in cases { - let message = rmp_serde::from_slice::(&bytes).unwrap_err().to_string(); - assert!(message.contains("Outcome"), "{label}: {message}"); + let message = rmp_serde::from_slice::(&bytes).unwrap_err().to_string(); + assert!(message.contains("UnitOutcome"), "{label}: {message}"); assert!(!message.contains("CANARY"), "{label}: {message}"); } } @@ -679,9 +724,9 @@ fn bounds_vectors() { let value = result.unwrap_or_else(|message| panic!("{file}: {message}")); assert_eq!(rmp_serde::to_vec(&value).unwrap(), vector(detail), "{file}"); match file { - "arm-key-int32-min.msgpack" => assert_eq!(value.outcomes.last(), Some(&Outcome::Floor)), + "arm-key-int32-min.msgpack" => assert_eq!(value.outcomes.last(), Some(&Outcome::Floor(id_arm("0000000000a4")))), "arm-key-int32-max.msgpack" | "arm-key-wide-coding.msgpack" => { - assert_eq!(value.outcomes.last(), Some(&Outcome::Ceiling)) + assert_eq!(value.outcomes.last(), Some(&Outcome::Ceiling(id_arm("0000000000a5")))) } _ => {} } @@ -715,7 +760,7 @@ fn bounds_vectors() { fn wrap_chain(wraps: usize, leaf: Node) -> UnionSurface { let mut node = leaf; for _ in 0..wraps { - node = Node::Wrap(WrapArm { inner: Some(std::boxed::Box::new(node)) }); + node = Node::Wrap(WrapArm { inner: Some(std::boxed::Box::new(node)), id: "0000000000b1".into() }); } UnionSurface { root: Some(node), ..constructed() } } @@ -724,9 +769,9 @@ fn wrap_chain(wraps: usize, leaf: Node) -> UnionSurface { // children) ends at 2 * (wraps + 1) + 2 and a Text leaf at 2 * (wraps + 1) + 1. #[test] fn the_constructed_chains_write_the_vector_bytes() { - let at_limit = wrap_chain(126, boxed("leaf", vec![])); + let at_limit = wrap_chain(126, boxed("0000000000f0", vec![])); assert_eq!(rmp_serde::to_vec(&at_limit).unwrap(), vector("depth-at-limit.msgpack")); - let past_limit = wrap_chain(127, text("leaf", CANARY)); + let past_limit = wrap_chain(127, text("0000000000f0", CANARY)); assert_eq!(rmp_serde::to_vec(&past_limit).unwrap(), vector("depth-past-limit.msgpack")); } @@ -744,14 +789,14 @@ fn write_refused_past_limit() { cache.register_entry_type::().unwrap(); cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&path)).unwrap(); - let at_limit = wrap_chain(126, boxed("leaf", vec![])); + let at_limit = wrap_chain(126, boxed("0000000000f0", vec![])); assert_eq!(cache.put("at", &at_limit).unwrap(), at_limit); let stored = cache.snapshot_envelopes(); assert_eq!(stored.len(), 1); assert_eq!(stored[0].payload, vector("depth-at-limit.msgpack")); let file_before = std::fs::read(&path).unwrap(); - let error = cache.put("past", &wrap_chain(127, text("leaf", CANARY))).unwrap_err(); + let error = cache.put("past", &wrap_chain(127, text("0000000000f0", CANARY))).unwrap_err(); assert_eq!(error.downcast_ref::(), Some(&PayloadTooDeep { depth: 257, limit: 256 })); assert!(!format!("{error:#}").contains(CANARY), "{error:#}"); assert_eq!(cache.get::("past").unwrap(), None); @@ -808,8 +853,8 @@ fn raw(type_name: &str, key: &str, payload: Vec) -> CultCacheEnvelope { #[test] fn the_raw_doors_hold_the_bound_for_arms_maps_and_arrays() { - let arms_at = rmp_serde::to_vec(&wrap_chain(126, boxed("leaf", vec![]))).unwrap(); - let arms_past = rmp_serde::to_vec(&wrap_chain(127, text("leaf", CANARY))).unwrap(); + let arms_at = rmp_serde::to_vec(&wrap_chain(126, boxed("0000000000f0", vec![]))).unwrap(); + let arms_past = rmp_serde::to_vec(&wrap_chain(127, text("0000000000f0", CANARY))).unwrap(); let maps_at = rmp_serde::to_vec_named(&map_doc(MAX_PAYLOAD_DEPTH - 1)).unwrap(); let maps_past = rmp_serde::to_vec_named(&map_doc(MAX_PAYLOAD_DEPTH)).unwrap(); let arrays_at = vector("depth-at-limit.msgpack"); @@ -857,13 +902,13 @@ fn the_union_chain_at_the_limit_decodes_on_a_1mib_thread() { .spawn(|| { assert_eq!( through_cache(&vector("depth-at-limit.msgpack")).unwrap(), - wrap_chain(126, boxed("leaf", vec![])) + wrap_chain(126, boxed("0000000000f0", vec![])) ); let refused = through_cache(&vector("depth-past-limit.msgpack")).unwrap_err(); assert!(refused.contains("the limit is 256"), "{refused}"); let mut cache = CultCache::new(); cache.register_entry_type::().unwrap(); - let value = wrap_chain(126, boxed("leaf", vec![])); + let value = wrap_chain(126, boxed("0000000000f0", vec![])); assert_eq!(cache.put("at", &value).unwrap(), value); assert_eq!(cache.get::("at").unwrap(), Some(value)); let mut deep = vec![0x91u8; 100_000]; diff --git a/packages/cultcache-ts/CHANGELOG.md b/packages/cultcache-ts/CHANGELOG.md index 6304d347..a2f10c54 100644 --- a/packages/cultcache-ts/CHANGELOG.md +++ b/packages/cultcache-ts/CHANGELOG.md @@ -3,6 +3,24 @@ All notable changes to `@gamecult/cultcache-ts` are recorded here, following [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). +## Unreleased + +### Breaking + +- A single-file store written as `cultcache.store.v3` is read and written: a record that holds an element id (an owned list element with an `[CultElementId]` member, carried through every write) marks the store v3, and a v3 store is carried through a rewrite with its ids. A runtime from before this change refuses a v3 store by name. +- A single-file store refuses a zero-byte file as `StoreUnreadableError`; only a path with nothing at it is an empty store. A header this runtime does not read is refused, and the refusal echoes the header only as `cultcache.store.v`. +- A write over a file the reader refuses (`StoreUnreadableError`, from `push`, `pushAll` and `delete`) refuses and leaves the file as it is. A flush used to replace such a file. A write set cannot merge with a disk it cannot read, so the file is repaired or removed by its owner before the store writes again. +- This reverses the 0.14.2 `pushAll` fix in two ways. A full `pushAll` over a file with no readable stamp no longer replaces it (healing it): the reader refuses it, and the file is left as it is. And a `cultcache.store.v3` stamp is read, not refused. The stamp rule has one owner, `refuseUnreadFormat`, asked only by the one reader that every write consults under the lock. The reader judges the stamp from the header before it decodes the body, so a newer store is refused by name whether or not its body is whole. +- A dangling symbolic link at the store path is an I/O error with code `ENOENT`, not an empty store. + +### Added + +- `readSingleFileStore(path)`, the one synchronous reader of a single-file store, for consumers that must read a store file rather than decode it themselves. `pullAll`, `push`, `delete` and `pushAll` ask the same reader. +- `StoreUnreadableError` for a store file this runtime cannot read, `SchemaConflictError` for a write the catalog cannot describe, and `schemaIdentityOf` with its result type `CultCacheSchemaIdentity` (`schemaId`, `schemaName`, `schemaVersion`, `compatibleSchemaIds`), the one derivation of a definition's identity. +- `SingleFileStoreRead`, exported: the result type of `readSingleFileStore` (`absent`, `format`, `envelopes`, `repairedLegacyPayload`). +- `CultCacheInspection.format` is `StoreFormat | "legacy.envelope-array"`: it reports `cultcache.store.v3` for a v3 store. +- A soft `pushAll` writes only where nothing is at the path, leaves a store it can read byte for byte, and refuses what the reader refuses. + ## [0.14.2] ### Added diff --git a/packages/cultcache-ts/src/cult-cache-inspector.ts b/packages/cultcache-ts/src/cult-cache-inspector.ts index 56a0f6e3..1c359157 100644 --- a/packages/cultcache-ts/src/cult-cache-inspector.ts +++ b/packages/cultcache-ts/src/cult-cache-inspector.ts @@ -1,12 +1,12 @@ import { decode } from "@msgpack/msgpack"; import { checkPayloadDepth } from "./payload-depth"; -import { STORE_FORMAT_VERSION, isV1Snapshot, requireV1RecordSlots } from "./store-format"; +import { type StoreFormat, isStoreSnapshot, refuseUnreadStoreBytes, requireStoreSlots, requireV1RecordSlots } from "./store-format"; export interface CultCacheInspection { filePath: string; fileSizeBytes: number; - format: "cultcache.store.v1" | "legacy.envelope-array"; + format: StoreFormat | "legacy.envelope-array"; catalog: InspectedCatalogEntry[]; records: InspectedRecord[]; } @@ -47,9 +47,10 @@ export function inspectCultCacheBytes( bytes: Uint8Array, fileSizeBytes = bytes.length, ): CultCacheInspection { + refuseUnreadStoreBytes(bytes); const decoded = decode(bytes); - if (isV1Snapshot(decoded)) { + if (isStoreSnapshot(decoded)) { return inspectV1Snapshot(filePath, fileSizeBytes, decoded); } @@ -60,7 +61,8 @@ export function inspectCultCacheBytes( throw new Error("CultCache file is not a recognized v1 snapshot or legacy envelope array."); } -function inspectV1Snapshot(filePath: string, fileSizeBytes: number, decoded: unknown[]): CultCacheInspection { +function inspectV1Snapshot(filePath: string, fileSizeBytes: number, decoded: [StoreFormat, ...unknown[]]): CultCacheInspection { + requireStoreSlots(decoded); const catalogRaw = decoded[1]; const recordsRaw = decoded[2]; if (!Array.isArray(catalogRaw) || !Array.isArray(recordsRaw)) { @@ -68,36 +70,25 @@ function inspectV1Snapshot(filePath: string, fileSizeBytes: number, decoded: unk } const catalog = catalogRaw.map(decodeCatalogEntry); + // An id names the entry that owns it; only an id no entry owns names the entry that lists it as compatible. const catalogBySchemaId = new Map(); for (const entry of catalog) { - catalogBySchemaId.set(entry.schemaId, entry); - for (const compatibleSchemaId of entry.compatibleSchemaIds) { - catalogBySchemaId.set(compatibleSchemaId, entry); + if (!catalogBySchemaId.has(entry.schemaId)) { + catalogBySchemaId.set(entry.schemaId, entry); } } - const records = recordsRaw.map((record) => decodeV1Record(record, catalogBySchemaId)); - for (const record of records) { - if (!catalog.some((entry) => entry.schemaId === record.schemaId)) { - const schemaVersion = inferSchemaVersionFromPayloadPreview(record.payloadPreview); - const schemaName = schemaVersion ? inferSchemaName(schemaVersion) : undefined; - if (schemaName) { - catalog.push({ - schemaId: record.schemaId, - schemaName, - schemaVersion: schemaVersion!, - contentHash: record.schemaId, - canonicalSchemaJson: "", - compatibleSchemaIds: [], - members: [], - }); + for (const entry of catalog) { + for (const compatibleSchemaId of entry.compatibleSchemaIds) { + if (!catalogBySchemaId.has(compatibleSchemaId)) { + catalogBySchemaId.set(compatibleSchemaId, entry); } } } - + const records = recordsRaw.map((record) => decodeV1Record(record, catalogBySchemaId)); return { filePath, fileSizeBytes, - format: STORE_FORMAT_VERSION, + format: decoded[0], catalog, records, }; @@ -205,64 +196,21 @@ function decodeV1Record(value: unknown, catalogBySchemaId: Map", + schemaName: catalogEntry.schemaName, storedAt, payloadBytes: payloadBytes.length, ...previewPayload(payloadBytes), }; } -function recoverCatalogEntry(schemaId: string, payload: Uint8Array): InspectedCatalogEntry | undefined { - let payloadPreview: unknown; - try { - payloadPreview = decodeBoundedPayload(payload); - } catch { - return undefined; - } - - const schemaVersion = inferSchemaVersionFromPayloadPreview(payloadPreview); - if (!schemaVersion) { - return undefined; - } - - const schemaName = inferSchemaName(schemaVersion); - if (!schemaName) { - return undefined; - } - - return { - schemaId, - schemaName, - schemaVersion, - contentHash: schemaId, - canonicalSchemaJson: "", - compatibleSchemaIds: [], - members: [], - }; -} - -function inferSchemaVersionFromPayloadPreview(payloadPreview: unknown): string | undefined { - if (Array.isArray(payloadPreview) && typeof payloadPreview[0] === "string") { - return payloadPreview[0]; - } - - if (isRecord(payloadPreview)) { - const schemaVersion = payloadPreview.schemaVersion ?? payloadPreview.schema_version; - return typeof schemaVersion === "string" ? schemaVersion : undefined; - } - - return undefined; -} - -function inferSchemaName(schemaVersion: string): string | undefined { - const match = /^(.*)\.v\d+$/.exec(schemaVersion); - return match?.[1]; -} - function decodeLegacyEnvelope(value: unknown): InspectedRecord { if (!isRecord(value)) { throw new Error("CultCache legacy envelopes must be MessagePack maps."); diff --git a/packages/cultcache-ts/src/cult-cache.ts b/packages/cultcache-ts/src/cult-cache.ts index 2f2fb063..88ab94e9 100644 --- a/packages/cultcache-ts/src/cult-cache.ts +++ b/packages/cultcache-ts/src/cult-cache.ts @@ -4,9 +4,11 @@ import { documentPayloadFromBytes, encodeDocumentPayload, resolveDocumentMembers, + schemaIdentityOf, } from "./document"; import { resolveDocumentFormatter } from "./document-formatter"; import type { DocumentPayload } from "./document"; +import { SchemaConflictError } from "./store-format"; import type { AnyCultCacheDocumentDefinition, CacheBackingStore, @@ -161,7 +163,9 @@ export class CultCache { } readonly #definitions = new Map(); + // The definition that owns each schema id, and every definition listing an id as compatible, in registration order. readonly #schemaIdDefinitions = new Map(); + readonly #listingDefinitions = new Map(); readonly #schemaNameDefinitions = new Map(); #hydrated = emptyHydratedState(); readonly #stores: CultCacheStoreRegistration[] = []; @@ -474,8 +478,10 @@ export class CultCache { type: envelope.type, payload, storedAt: envelope.storedAt, - schemaId: envelope.schemaId ?? registered.catalogEntry.schemaId, - catalogEntry: envelope.catalogEntry ?? registered.catalogEntry, + // The record is stamped with its registered schema's id, whatever id the envelope arrived under, so the + // catalog entry the store writes is the one its schema id names. + schemaId: registered.catalogEntry.schemaId, + catalogEntry: registered.catalogEntry, }; await this.#writeNow(registered, entry, parsed); return parsed; @@ -535,9 +541,13 @@ export class CultCache { } #registerNow(definition: AnyCultCacheDocumentDefinition): void { + // Every registration refusal is a SchemaConflictError naming the new definition's schema id and both schema names. + const refuse = (holder: RegisteredDefinition, claimed: CultCacheSchemaCatalogEntry, what: string) => + new SchemaConflictError(claimed.schemaId, [holder.catalogEntry.schemaName, claimed.schemaName], "", + `CultCache ${what} is already registered for type "${holder.definition.type}" and cannot also be claimed by type "${definition.type}".`); const existing = this.#definitions.get(definition.type); if (existing && existing.definition !== definition) { - throw new Error(`CultCache already has a different definition registered for type "${definition.type}".`); + throw refuse(existing, this.#createCatalogEntry(definition), `type "${definition.type}"`); } const registered: RegisteredDefinition = existing ?? { @@ -557,25 +567,28 @@ export class CultCache { indexAccessors.set(indexName, this.#compileAccessor(definition.type, `index "${indexName}"`, accessor)); } - const schemaIds = registered.catalogEntry.compatibleSchemaIds ?? [registered.catalogEntry.schemaId]; - for (const schemaId of schemaIds) { - const existingBySchema = this.#schemaIdDefinitions.get(schemaId); - if (existingBySchema && existingBySchema !== registered) { - throw new Error(`CultCache schema id "${schemaId}" is already registered for type "${existingBySchema.definition.type}".`); - } + // One definition owns each schema id. Others may list it as compatible beside its owner: a record under the id resolves to + // its owner, and to a lister only when nothing owns it. Neither depends on the order definitions were registered in. + const ownId = registered.catalogEntry.schemaId; + const listed = (registered.catalogEntry.compatibleSchemaIds ?? []).filter((schemaId) => schemaId !== ownId); + const owner = this.#schemaIdDefinitions.get(ownId); + if (owner && owner !== registered) { + throw refuse(owner, registered.catalogEntry, `schema id "${ownId}"`); } const existingBySchemaName = this.#schemaNameDefinitions.get(registered.catalogEntry.schemaName); if (existingBySchemaName && existingBySchemaName !== registered) { - throw new Error( - `CultCache schema name "${registered.catalogEntry.schemaName}" is already registered for type "${existingBySchemaName.definition.type}".`, - ); + throw refuse(existingBySchemaName, registered.catalogEntry, `schema name "${registered.catalogEntry.schemaName}"`); } const derived = this.#deriveTypeLookups(definition.type, nameAccessor, indexAccessors); // Every check and accessor has run; install. this.#definitions.set(definition.type, registered); - for (const schemaId of schemaIds) { - this.#schemaIdDefinitions.set(schemaId, registered); + this.#schemaIdDefinitions.set(ownId, registered); + for (const schemaId of listed) { + const listers = this.#listingDefinitions.get(schemaId) ?? []; + if (!listers.includes(registered)) { + this.#listingDefinitions.set(schemaId, [...listers, registered]); + } } this.#schemaNameDefinitions.set(registered.catalogEntry.schemaName, registered); this.#installAccessors(registered, nameAccessor, indexAccessors, derived); @@ -698,8 +711,7 @@ export class CultCache { } #createCatalogEntry(definition: AnyCultCacheDocumentDefinition): CultCacheSchemaCatalogEntry { - const schemaName = definition.schemaName ?? definition.type; - const schemaVersion = definition.schemaVersion ?? `${schemaName}.v1`; + const { schemaId, schemaName, schemaVersion, compatibleSchemaIds } = schemaIdentityOf(definition); const canonicalSchemaJson = definition.canonicalSchemaJson ?? JSON.stringify({ schemaName, schemaVersion, @@ -716,10 +728,6 @@ export class CultCache { isName: member.isName === true, })), }); - const schemaId = definition.schemaId ?? schemaName; - const compatibleSchemaIds = [ - ...new Set([schemaId, ...(definition.compatibleSchemaIds ?? [])]), - ]; return { schemaId, @@ -727,7 +735,7 @@ export class CultCache { schemaVersion, contentHash: definition.contentHash ?? schemaId, canonicalSchemaJson, - compatibleSchemaIds, + compatibleSchemaIds: [...compatibleSchemaIds], members: [...resolveDocumentMembers(definition)] .sort((left, right) => left.slot - right.slot) .map((member) => ({ @@ -810,7 +818,16 @@ export class CultCache { #resolveDefinitionForEnvelope(entry: CultCacheEnvelope): RegisteredDefinition | undefined { if (entry.schemaId) { - return this.#schemaIdDefinitions.get(entry.schemaId) + const listers = this.#listingDefinitions.get(entry.schemaId) ?? []; + const owner = this.#schemaIdDefinitions.get(entry.schemaId); + if (!owner && listers.length > 1) { + const types = listers.map((lister) => lister.definition.type).sort(); + throw new SchemaConflictError(entry.schemaId, listers.map((lister) => lister.catalogEntry.schemaName).sort(), entry.key, + `CultCache schema id "${entry.schemaId}" of record "${entry.key}" is owned by no registered type and declared compatible by ` + + `several (${types.join(", ")}), so it resolves to none. Register the type that owns the id, or declare it on one type.`); + } + return owner + ?? listers[0] ?? this.#schemaNameDefinitions.get(entry.type) ?? this.#definitions.get(entry.type); } @@ -880,8 +897,10 @@ export class CultCache { return { ...entry, payload: this.#cloneBytes(entry.payload), - schemaId: entry.schemaId ?? registered.catalogEntry.schemaId, - catalogEntry: entry.catalogEntry ?? registered.catalogEntry, + // Admitted under the registered schema's id, as a put stamps it: a record loaded under a compatible id is + // written back under the id its catalog entry carries. + schemaId: registered.catalogEntry.schemaId, + catalogEntry: registered.catalogEntry, value, lookups: this.#lookupsFor(registered.nameAccessor, registered.indexAccessors, value), }; diff --git a/packages/cultcache-ts/src/document.ts b/packages/cultcache-ts/src/document.ts index 0b019011..99ec170c 100644 --- a/packages/cultcache-ts/src/document.ts +++ b/packages/cultcache-ts/src/document.ts @@ -149,6 +149,27 @@ export function resolveDocumentMembers(definition: AnyCultCacheDocumentDefinitio })); } +// The identity a definition's records carry: the schema id they are stamped with, the schema name and version, and every id +// the definition answers to (its own first). Every CultCache runtime surface and every CultNet or CultMesh layer over it +// derives a definition's identity here and nowhere else, so a record's id means the same type wherever it is read. +export interface CultCacheSchemaIdentity { + readonly schemaId: string; + readonly schemaName: string; + readonly schemaVersion: string; + readonly compatibleSchemaIds: readonly string[]; +} + +export function schemaIdentityOf(definition: AnyCultCacheDocumentDefinition): CultCacheSchemaIdentity { + const schemaName = definition.schemaName ?? definition.type; + const schemaId = definition.schemaId ?? schemaName; + return { + schemaId, + schemaName, + schemaVersion: definition.schemaVersion ?? `${schemaName}.v1`, + compatibleSchemaIds: [...new Set([schemaId, ...(definition.compatibleSchemaIds ?? [])])], + }; +} + export function defineDocumentRegistry< TDefinitions extends readonly AnyCultCacheDocumentDefinition[], >(...definitions: TDefinitions): CultCacheDocumentRegistry { diff --git a/packages/cultcache-ts/src/index.ts b/packages/cultcache-ts/src/index.ts index 8646d12e..1facf6e2 100644 --- a/packages/cultcache-ts/src/index.ts +++ b/packages/cultcache-ts/src/index.ts @@ -9,10 +9,13 @@ export { documentPayloadBytes, documentPayloadFromBytes, encodeDocumentPayload, + schemaIdentityOf, + type CultCacheSchemaIdentity, } from "./document"; export { CultCachePayloadDepthError, MAX_PAYLOAD_DEPTH } from "./payload-depth"; export type { DocumentPayload } from "./document"; -export { SingleFileMessagePackBackingStore } from "./single-file-messagepack-backing-store"; +export { SingleFileMessagePackBackingStore, readSingleFileStore, type SingleFileStoreRead } from "./single-file-messagepack-backing-store"; +export { SchemaConflictError, StoreUnreadableError } from "./store-format"; export type { AnyCultCacheDocumentDefinition, CacheBackingStore, diff --git a/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts b/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts index f0641e66..2501c632 100644 --- a/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts +++ b/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts @@ -1,13 +1,13 @@ -import { lstat, mkdir, readFile } from "node:fs/promises"; +import { lstatSync, readFileSync } from "node:fs"; +import { mkdir } from "node:fs/promises"; import { dirname, resolve } from "node:path"; import { decode, encode } from "@msgpack/msgpack"; import { z } from "zod"; import { documentPayloadBytes, documentPayloadFromBytes } from "./document"; -import { checkPayloadDepth } from "./payload-depth"; import { commitStoreFile, withStoreWriteLock } from "./store-file"; -import { STORE_FORMAT_VERSION, isV1Snapshot, refuseUnreadStoreBytes, requireV1RecordSlots } from "./store-format"; +import { STORE_FORMAT_VERSION, SchemaConflictError, type StoreFormat, StoreUnreadableError, isStoreSnapshot, refuseUnreadStoreBytes, requireStoreSlots, requireV1RecordSlots } from "./store-format"; import type { CacheBackingStore, CultCacheEnvelope, @@ -39,116 +39,89 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore { #writeQueue: Promise = Promise.resolve(); + // The header this store read; a rewrite keeps it, so a store that holds element ids never sheds its marker. + #format: StoreFormat = STORE_FORMAT_VERSION; + constructor(filePath: string) { this.filePath = resolve(filePath); } async pullAll(): Promise { - const { envelopes, healLegacy } = await this.#readDisk(); - if (healLegacy) { + // The disk decides the header: a file that is gone or legacy is not marked. + this.#format = STORE_FORMAT_VERSION; + const disk = this.#readDisk(); + this.#format = disk.format; + if (disk.repairedLegacyPayload) { // The heal decides from its own read, taken under the store lock. await this.#enqueue(async () => { - const again = await this.#readDisk(); - if (again.healLegacy) { - await this.#writeAll(again.envelopes); + const again = this.#readDisk(); + this.#format = again.format; + if (again.repairedLegacyPayload) { + await this.#writeAll([], again.envelopes); } }); } - return envelopes; - } - - async #readDisk(): Promise<{ envelopes: CultCacheEnvelope[]; healLegacy: boolean }> { - // An empty file is not a store, since no CultCache writer leaves one, so it is - // decoded and refused. Only a read that finds nothing at the path is an empty store. - const data = await readStore(this.filePath); - if (data === undefined) { - return { envelopes: [], healLegacy: false }; - } - - const decoded = decode(data); - const snapshot = decodeSnapshot(decoded); - if (snapshot) { - const envelopes = snapshot.records.map((record) => { - const catalogEntry = resolveCatalogEntryForRecord(record, snapshot.catalogBySchemaId); - if (!catalogEntry) { - throw new Error(`CultCache persisted record "${record.key}" references missing schema id "${record.schemaId}".`); - } - - return { - key: record.key, - type: catalogEntry.schemaName, - schemaId: record.schemaId, - catalogEntry, - payload: documentPayloadFromBytes(record.payload), - storedAt: record.storedAt, - }; - }); - return { envelopes, healLegacy: false }; - } - - const legacy = decodeLegacyEnvelopeArray(decoded); - if (legacy) { - let repairedLegacyPayload = false; - const normalized = legacy.map((entry) => { - const payload = normalizePayload(entry.payload); - if (payload !== entry.payload) { - repairedLegacyPayload = true; - } - - return { - ...entry, - payload, - }; - }); - const parsed: CultCacheEnvelope[] = envelopeArraySchema.parse(normalized).map((entry) => ({ - ...entry, - payload: documentPayloadFromBytes(entry.payload), - })); - - return { envelopes: parsed, healLegacy: repairedLegacyPayload }; - } - - throw new Error(`CultCache file ${this.filePath} is not a recognized CultCache MessagePack store.`); + return disk.envelopes; } async push(entry: CultCacheEnvelope): Promise { await this.#enqueue(async () => { - const existing = (await this.#readDisk()).envelopes; + const disk = this.#readDisk(); + this.#format = disk.format; + const existing = disk.envelopes; const filtered = existing.filter( (candidate) => !(candidate.type === entry.type && candidate.key === entry.key), ); - filtered.push(entry); - await this.#writeAll(filtered); + await this.#writeAll([entry], filtered); }); } async delete(entry: CultCacheEnvelope): Promise { await this.#enqueue(async () => { - const existing = (await this.#readDisk()).envelopes; + const disk = this.#readDisk(); + this.#format = disk.format; + const existing = disk.envelopes; const filtered = existing.filter( (candidate) => !(candidate.type === entry.type && candidate.key === entry.key), ); - await this.#writeAll(filtered); + await this.#writeAll([], filtered); }); } async pushAll(entries: CultCacheEnvelope[], options: PushAllOptions = {}): Promise { await this.#enqueue(async () => { - // A whole write of a file it cannot read proceeds as before (a dangling link is - // replaced, R3); only a stamp it can read is judged. - const data = options.soft ? await readStore(this.filePath) : await readStore(this.filePath).catch(() => undefined); - if (data !== undefined) { - if (options.soft) { + // A flush of the whole store writes the header the file on disk carries, read now: a file marked for element ids + // stays marked, and one that is not (or is gone or legacy) is written unmarked. The file is read by the same + // reader as `pullAll`, so one it would refuse (not exactly one store, a variant, a body it cannot decode) is refused + // and left as it is. A soft flush asks that reader too: nothing at the path is written, a store is left alone. + try { + const disk = this.#readDisk(); + if (options.soft && !disk.absent) { return; } - refuseUnreadStoreBytes(data); + + this.#format = disk.format; + } catch (error) { + // Only a dangling link reaches here with ENOENT: it holds no store, so a whole-store flush has no header to keep and + // replaces the link (R3 pins this). A soft flush does not replace what it cannot read, and every merging writer + // has already refused the link when it read the current entries. + if (options.soft || (error as NodeJS.ErrnoException).code !== "ENOENT") { + throw error; + } + + this.#format = STORE_FORMAT_VERSION; } - await this.#writeAll(entries); + await this.#writeAll(entries, []); }); } + // The one reader of the store file, asked by open, push, delete and flush alike; see readSingleFileStore. + #readDisk(): SingleFileStoreRead { + return readSingleFileStore(this.filePath); + } + async #enqueue(operation: () => Promise): Promise { let result!: T; @@ -167,29 +140,54 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore { return result; } - async #writeAll(entries: CultCacheEnvelope[]): Promise { + async #writeAll(supplied: CultCacheEnvelope[], arrived: CultCacheEnvelope[]): Promise { await mkdir(dirname(this.filePath), { recursive: true }); - await commitStoreFile(this.filePath, encode(encodeSnapshot(entries))); + await commitStoreFile(this.filePath, encode(encodeSnapshot(supplied, arrived, this.#format))); } } /** - * Reads a store file, or returns undefined when nothing is at its path. A dangling symbolic - * link is something: reading it as empty would let the writer rename a file over the link - * and move the store off its volume. Any other failure to reach the file is thrown. + * The one reader of a CultCache single-file store, for this package's store and for any consumer that must read a + * store file (an authority bundle, say) rather than decode it itself. A path with nothing at it is an absent, empty, + * unmarked store. A dangling symbolic link is something: reading it as empty would let a writer rename a file over + * the link and move the store off its volume, so it throws with code ENOENT, as does any other failure to reach the + * file. A zero-byte file is not a store and, like anything that does not decode completely, throws StoreUnreadableError. + * The read is synchronous so a consumer that loads authority at start-up needs no second reader to stay synchronous. */ -async function readStore(path: string): Promise { +export function readSingleFileStore(path: string): SingleFileStoreRead { + const data = readStoreBytes(path); + if (data === undefined) { + return { absent: true, format: STORE_FORMAT_VERSION, envelopes: [], repairedLegacyPayload: false }; + } + try { - return await readFile(path); + return { absent: false, ...decodeStoreFile(data) }; + } catch (error) { + throw new StoreUnreadableError(path, error); + } +} + +/** + * The bytes at `path`, or undefined when nothing is there. Nothing is there only when both looks agree: the read found no + * file and the follow-up lstat found no entry. Every other outcome is an error, because the path may have changed + * between the two calls (a link in a parent swapped for a file, a store created or removed): an lstat failure other than + * ENOENT is thrown as it is, and an entry the lstat does find, a dangling link or one that appeared since the read, is + * thrown with code ENOENT rather than read as an empty store. `lstat` is a seam so a test can stage the second look; + * this module exports the function for that test, and the package index does not. + */ +export function readStoreBytes(path: string, lstat: (path: string) => { isSymbolicLink(): boolean } = lstatSync): Uint8Array | undefined { + try { + return readFileSync(path); } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") { throw error; } } + let entry: { isSymbolicLink(): boolean }; try { - await lstat(path); + entry = lstat(path); } catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") { return undefined; @@ -198,7 +196,8 @@ async function readStore(path: string): Promise { throw error; } - throw Object.assign(new Error(`CultCache store ${path} is a symbolic link whose target does not exist.`), { code: "ENOENT" }); + const what = entry.isSymbolicLink() ? "is a symbolic link whose target does not exist" : "appeared while it was being read"; + throw Object.assign(new Error(`CultCache store ${path} ${what}.`), { code: "ENOENT" }); } type PersistedRecord = { @@ -209,13 +208,19 @@ type PersistedRecord = { }; type DecodedSnapshot = { + format: StoreFormat; catalogBySchemaId: Map; records: PersistedRecord[]; }; -function encodeSnapshot(entries: CultCacheEnvelope[]): unknown[] { - const catalog = [...catalogEntriesFor(entries).values()] - .sort((left, right) => compareOrdinal(left.schemaName, right.schemaName)); +function encodeSnapshot(supplied: CultCacheEnvelope[], arrived: CultCacheEnvelope[], format: StoreFormat): unknown[] { + const replaced = new Set(supplied.map((entry) => `${entry.type}::${entry.key}`)); + const kept = arrived.filter((entry) => !replaced.has(`${entry.type}::${entry.key}`)); + const entries = [...kept, ...supplied]; + const catalog = catalogEntriesFor([ + ...kept.map((envelope) => ({ envelope, registered: false })), + ...supplied.map((envelope) => ({ envelope, registered: true })), + ]).sort((left, right) => compareOrdinal(left.schemaName, right.schemaName) || compareOrdinal(left.schemaId, right.schemaId)); const records = [...entries] .sort((left, right) => compareOrdinal(left.key, right.key)) .map((entry) => [ @@ -226,34 +231,134 @@ function encodeSnapshot(entries: CultCacheEnvelope[]): unknown[] { ]); return [ - STORE_FORMAT_VERSION, + format, catalog.map(encodeCatalogEntry), records, ]; } -function catalogEntriesFor(entries: CultCacheEnvelope[]): Map { - const catalog = new Map(); - for (const entry of entries) { - const schemaId = schemaIdForEnvelope(entry); - if (!catalog.has(schemaId)) { - catalog.set(schemaId, entry.catalogEntry ?? { +// The catalog a write leaves, derived from the records being written and nothing else. For each schema id a record carries, one +// entry that publishes it: the entry that owns the id (a registered one, else one that arrived with the records), else one that +// lists it as compatible (registered first). Entries are written as chosen, never merged; a registered entry wins over an arrived +// one with the same own id. Records of different types under one id, two arrived entries with the same own id and different schema +// names, or a record no chosen entry publishes, refuse the write. An envelope the caller supplied is registered; one read back +// from the file is arrived. +function catalogEntriesFor( + records: Array<{ envelope: CultCacheEnvelope; registered: boolean }>, +): CultCacheSchemaCatalogEntry[] { + const entries: Array<{ entry: CultCacheSchemaCatalogEntry; registered: boolean }> = []; + const byId = new Map(); + const tiers = new Map>(); + for (const { envelope, registered } of records) { + const schemaId = schemaIdForEnvelope(envelope); + const supplied = envelope.catalogEntry; + const publishes = + supplied !== undefined && (supplied.schemaId === schemaId || (supplied.compatibleSchemaIds ?? []).includes(schemaId)); + const entry = publishes && supplied ? supplied : defaultEntryFor(envelope); + entries.push({ entry, registered }); + byId.set(schemaId, [...(byId.get(schemaId) ?? []), envelope]); + tiers.set(schemaId, [...(tiers.get(schemaId) ?? []), { envelope, registered, entry }]); + } + + const lists = (entry: CultCacheSchemaCatalogEntry, schemaId: string): boolean => + (entry.compatibleSchemaIds ?? [entry.schemaId]).includes(schemaId); + const chosen = new Map(); + for (const schemaId of [...byId.keys()].sort(compareOrdinal)) { + const carrying = byId.get(schemaId)!; + const recordKey = carrying.map((envelope) => envelope.key).sort(compareOrdinal)[0]!; + // Records of one tier under one id are one type. Across tiers the id keeps the type its records resolve to, so a write may + // not retype records that are already there: unless the entries of both tiers own the id, which is a rename of the schema + // under a stable id, and the registered descriptor wins. A read-back record's type is its schema's name, so tiers compare + // the names of the entries that publish the id. + const carried = tiers.get(schemaId)!; + for (const tier of [true, false]) { + const types = [...new Set(carried.filter((candidate) => candidate.registered === tier).map((candidate) => candidate.envelope.type))].sort( + compareOrdinal, + ); + if (types.length > 1) { + throw new SchemaConflictError(schemaId, types, recordKey); + } + } + + const tierNames = (tier: boolean): string[] => [ + ...new Set(carried.filter((candidate) => candidate.registered === tier).map((candidate) => candidate.entry.schemaName)), + ]; + const ownedBy = (tier: boolean): boolean => + carried.some((candidate) => candidate.registered === tier && candidate.entry.schemaId === schemaId); + const [registeredName, arrivedName] = [tierNames(true)[0], tierNames(false)[0]]; + if (registeredName !== undefined && arrivedName !== undefined && registeredName !== arrivedName && !(ownedBy(true) && ownedBy(false))) { + throw new SchemaConflictError(schemaId, [registeredName, arrivedName].sort(compareOrdinal), recordKey); + } + + const ownRegistered = canonical(entries.filter((candidate) => candidate.registered && candidate.entry.schemaId === schemaId))[0]; + const ownArrived = canonical(entries.filter((candidate) => !candidate.registered && candidate.entry.schemaId === schemaId)); + // Entries of one tier that share an own id must be one schema; a registered entry settles a disagreement among arrived ones. + const ownRegisteredAll = canonical(entries.filter((candidate) => candidate.registered && candidate.entry.schemaId === schemaId)); + const names = [ + ...new Set((ownRegisteredAll.length > 0 ? ownRegisteredAll : ownArrived).map((candidate) => candidate.entry.schemaName)), + ].sort(compareOrdinal); + if (names.length > 1) { + throw new SchemaConflictError(schemaId, names, recordKey); + } + + let pick: { entry: CultCacheSchemaCatalogEntry; registered: boolean } | undefined; + if (ownRegistered !== undefined) { + pick = ownRegistered; + } else if (ownArrived.length > 0) { + pick = ownArrived[0]; + } else { + pick = + canonical(entries.filter((candidate) => candidate.registered && lists(candidate.entry, schemaId)))[0] ?? + canonical(entries.filter((candidate) => !candidate.registered && lists(candidate.entry, schemaId)))[0]; + } + + if (pick === undefined) { + continue; + } + + const prior = chosen.get(pick.entry.schemaId); + if (prior !== undefined && prior.registered && !pick.registered) { + continue; + } + + chosen.set(pick.entry.schemaId, pick); + } + + for (const [schemaId, carrying] of byId) { + if (![...chosen.values()].some(({ entry }) => entry.schemaId === schemaId || lists(entry, schemaId))) { + throw new SchemaConflictError( schemaId, - schemaName: entry.type, - schemaVersion: `${entry.type}.v1`, - contentHash: schemaId, - canonicalSchemaJson: JSON.stringify({ - schemaName: entry.type, - schemaVersion: `${entry.type}.v1`, - members: [], - }), - compatibleSchemaIds: [schemaId], - members: [], - }); + [...chosen.values()].map(({ entry }) => entry.schemaName).sort(compareOrdinal), + carrying.map((envelope) => envelope.key).sort(compareOrdinal)[0]!, + ); } } - return catalog; + return [...chosen.values()].map(({ entry }) => entry); +} + +// Entries that tie, in one fixed order, so the catalog does not depend on the order they arrive in. +function canonical(candidates: T[]): T[] { + const key = (candidate: T): string => + [candidate.entry.schemaName, candidate.entry.contentHash ?? "", (candidate.entry.compatibleSchemaIds ?? []).join(",")].join("\u0000"); + return [...candidates].sort((left, right) => compareOrdinal(key(left), key(right))); +} + +function defaultEntryFor(entry: CultCacheEnvelope): CultCacheSchemaCatalogEntry { + const schemaId = schemaIdForEnvelope(entry); + return { + schemaId, + schemaName: entry.type, + schemaVersion: `${entry.type}.v1`, + contentHash: schemaId, + canonicalSchemaJson: JSON.stringify({ + schemaName: entry.type, + schemaVersion: `${entry.type}.v1`, + members: [], + }), + compatibleSchemaIds: [schemaId], + members: [], + }; } function schemaIdForEnvelope(entry: CultCacheEnvelope): string { @@ -285,82 +390,89 @@ function encodeCatalogMember(member: CultCacheSchemaCatalogMember): unknown[] { ]; } +export type SingleFileStoreRead = { + // True when nothing is at the path; the other fields then describe an empty unmarked store. + absent: boolean; + format: StoreFormat; + envelopes: CultCacheEnvelope[]; + // A legacy file whose payloads were not bytes: reading it repairs them, and the store rewrites the file. + repairedLegacyPayload: boolean; +}; + +function decodeStoreFile(data: Uint8Array): Omit { + // The stamp is judged before the body is decoded, as Rust's store_header does: a newer store is refused by name. + refuseUnreadStoreBytes(data); + const decoded = decode(data); + const snapshot = decodeSnapshot(decoded); + if (snapshot) { + const envelopes = snapshot.records.map((record): CultCacheEnvelope => { + const catalogEntry = snapshot.catalogBySchemaId.get(record.schemaId); + if (!catalogEntry) { + throw new Error(`CultCache persisted record "${record.key}" references missing schema id "${record.schemaId}".`); + } + + return { + key: record.key, + type: catalogEntry.schemaName, + schemaId: record.schemaId, + catalogEntry, + payload: documentPayloadFromBytes(record.payload), + storedAt: record.storedAt, + }; + }); + return { format: snapshot.format, envelopes, repairedLegacyPayload: false }; + } + + const legacy = decodeLegacyEnvelopeArray(decoded); + if (!legacy) { + throw new Error("CultCache file is not a recognized CultCache MessagePack store."); + } + + let repairedLegacyPayload = false; + const normalized = legacy.map((entry) => { + const payload = normalizePayload(entry.payload); + if (payload !== entry.payload) { + repairedLegacyPayload = true; + } + + return { + ...entry, + payload, + }; + }); + return { format: STORE_FORMAT_VERSION, envelopes: envelopeArraySchema.parse(normalized).map((entry) => ({ ...entry, payload: documentPayloadFromBytes(entry.payload) })) as CultCacheEnvelope[], repairedLegacyPayload }; +} + function decodeSnapshot(decoded: unknown): DecodedSnapshot | undefined { - if (!isV1Snapshot(decoded)) { + if (!isStoreSnapshot(decoded)) { return undefined; } + requireStoreSlots(decoded); const catalogRaw = decoded[1]; const recordsRaw = decoded[2]; if (!Array.isArray(catalogRaw) || !Array.isArray(recordsRaw)) { throw new Error("CultCache v1 snapshot must contain a schema catalog and record array."); } + // A schema id names the entry that has it as its own id; only an id no entry owns names the entry that lists it as compatible. + const catalogEntries = catalogRaw.map(decodeCatalogEntry); const catalogBySchemaId = new Map(); - for (const entry of catalogRaw) { - const catalogEntry = decodeCatalogEntry(entry); - catalogBySchemaId.set(catalogEntry.schemaId, catalogEntry); + for (const catalogEntry of catalogEntries) { + if (!catalogBySchemaId.has(catalogEntry.schemaId)) { + catalogBySchemaId.set(catalogEntry.schemaId, catalogEntry); + } + } + for (const catalogEntry of catalogEntries) { for (const compatibleSchemaId of catalogEntry.compatibleSchemaIds ?? []) { - catalogBySchemaId.set(compatibleSchemaId, catalogEntry); + if (!catalogBySchemaId.has(compatibleSchemaId)) { + catalogBySchemaId.set(compatibleSchemaId, catalogEntry); + } } } const records = recordsRaw.map(decodeRecord); - return { catalogBySchemaId, records }; -} - -function resolveCatalogEntryForRecord( - record: PersistedRecord, - catalogBySchemaId: Map, -): CultCacheSchemaCatalogEntry | undefined { - const existing = catalogBySchemaId.get(record.schemaId); - if (existing) { - return existing; - } - - const schemaVersion = inferSchemaVersionFromPayload(record.payload); - if (!schemaVersion) { - return undefined; - } - - const schemaName = inferSchemaName(schemaVersion); - if (!schemaName) { - return undefined; - } - - return { - schemaId: record.schemaId, - schemaName, - schemaVersion, - contentHash: record.schemaId, - canonicalSchemaJson: "", - compatibleSchemaIds: [], - members: [], - }; -} - -function inferSchemaVersionFromPayload(payload: Uint8Array): string | undefined { - try { - checkPayloadDepth(payload); - const decoded = decode(payload); - if (Array.isArray(decoded) && typeof decoded[0] === "string") { - return decoded[0]; - } - - if (isObject(decoded)) { - const schemaVersion = decoded.schemaVersion ?? decoded.schema_version; - return typeof schemaVersion === "string" ? schemaVersion : undefined; - } - - return undefined; - } catch { - return undefined; - } -} - -function inferSchemaName(schemaVersion: string): string | undefined { - const match = /^(.*)\.v\d+$/.exec(schemaVersion); - return match?.[1]; + return { format: decoded[0], catalogBySchemaId, records }; } function decodeCatalogEntry(value: unknown): CultCacheSchemaCatalogEntry { diff --git a/packages/cultcache-ts/src/store-format.ts b/packages/cultcache-ts/src/store-format.ts index 197dc659..4efab504 100644 --- a/packages/cultcache-ts/src/store-format.ts +++ b/packages/cultcache-ts/src/store-format.ts @@ -2,26 +2,33 @@ import { decode } from "@msgpack/msgpack"; export const STORE_FORMAT_VERSION = "cultcache.store.v1"; +/** + * A store that can hold element ids. A reader older than element ids refuses this header: it would skip the id slots + * of a nested element and rewrite the element without them. This runtime writes it back only for a store it read + * under it; it does not decide when a document carries ids. + */ +export const STORE_FORMAT_ELEMENT_IDS = "cultcache.store.v3"; + +export type StoreFormat = typeof STORE_FORMAT_VERSION | typeof STORE_FORMAT_ELEMENT_IDS; + const STORE_FORMAT_PREFIX = "cultcache.store."; const PERSISTED_RECORD_SLOTS = 4; +const STORE_SLOTS = 3; /** - * True for a v1 snapshot, false for anything that is not a CultCache store header (the legacy - * envelope array). Any other `cultcache.store.*` header is refused by name. + * True for a decoded array that leads with a `cultcache.store.*` stamp, false for the legacy envelope array. It judges the + * shape only: the verdict on the stamp is {@link refuseUnreadStoreBytes}, which a reader calls on the bytes before it decodes, + * so the stamp of a decoded array it is given is one this runtime reads. */ -export function isV1Snapshot(decoded: unknown): decoded is unknown[] { - if (!Array.isArray(decoded) || typeof decoded[0] !== "string" || !decoded[0].startsWith(STORE_FORMAT_PREFIX)) { - return false; - } - - refuseUnreadFormat(decoded[0]); - return true; +export function isStoreSnapshot(decoded: unknown): decoded is [StoreFormat, ...unknown[]] { + return Array.isArray(decoded) && typeof decoded[0] === "string" && decoded[0].startsWith(STORE_FORMAT_PREFIX); } /** - * Refuses a store whose header names a format this runtime does not read, judging the header - * alone: the leading array's first element, when it is a complete string. Bytes with no such - * header (a legacy array, an empty, truncated or garbled file) are not refused here. + * Refuses a store whose header names a format this runtime does not read, judging the header alone: the leading array's first + * element, when it is a complete string. The reader calls it before it decodes the body, so a newer store is refused by name + * whether or not its body is whole. Bytes with no such header (a legacy array, an empty, truncated or garbled file) are not + * refused here. */ export function refuseUnreadStoreBytes(bytes: Uint8Array): void { const header = peekHeader(bytes); @@ -30,17 +37,8 @@ export function refuseUnreadStoreBytes(bytes: Uint8Array): void { } } -function refuseUnreadFormat(header: string): void { - if (header !== STORE_FORMAT_VERSION) { - throw new Error( - `CultCache store format ${describeHeader(header)} is not one this runtime reads; it reads "${STORE_FORMAT_VERSION}" only.`, - ); - } -} - -// The first element of a leading non-empty MessagePack array when it is a complete string; -// anything else is undefined. The string is decoded by the decoder a read uses, so a stamp -// gets one verdict from a read and from a whole write. +// The first element of a leading non-empty MessagePack array when it is a complete string; anything else is undefined. The +// string is decoded by the decoder a read uses, so a stamp gets one verdict from the peek and from the whole decode. function peekHeader(bytes: Uint8Array): string | undefined { const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); const lead = bytes[0]; @@ -64,6 +62,50 @@ function peekHeader(bytes: Uint8Array): string | undefined { return typeof header === "string" ? header : undefined; } +/** The one verdict on a stamp: this runtime reads v1 and v3, and any other `cultcache.store.*` header is refused by name. */ +function refuseUnreadFormat(header: string): void { + if (header !== STORE_FORMAT_VERSION && header !== STORE_FORMAT_ELEMENT_IDS) { + throw new Error( + `CultCache store format ${describeHeader(header)} is not one this runtime reads; it reads "${STORE_FORMAT_VERSION}" and "${STORE_FORMAT_ELEMENT_IDS}" only.`, + ); + } +} + +/** + * A write the catalog cannot describe: records of different types under one schema id, two arrived schemas that share an id and + * disagree on the schema name, or a record no chosen entry publishes. Nothing is written. It names the id, the names involved and + * a record key. A cache throws it too, with an empty record key, when a definition cannot register beside one it already holds, + * and when a record's schema id names no single registered definition. + */ +export class SchemaConflictError extends Error { + readonly schemaId: string; + readonly schemaNames: string[]; + readonly recordKey: string; + + constructor(schemaId: string, schemaNames: string[], recordKey: string, message?: string) { + super(message ?? `Schema id "${schemaId}" cannot describe record "${recordKey}": ${schemaNames.map((name) => `"${name}"`).join(", ")}.`); + this.name = "SchemaConflictError"; + this.schemaId = schemaId; + this.schemaNames = schemaNames; + this.recordKey = recordKey; + } +} + +/** + * A store file this runtime cannot read: not exactly one complete store (truncated, bytes after it, a missing or extra slot), + * a header or record it does not know, or a body it cannot decode. Open, flush and every rewrite refuse a file with this + * error, and a refused file is left as it was. `cause` is what the reader choked on. + */ +export class StoreUnreadableError extends Error { + readonly filePath: string; + + constructor(filePath: string, cause: unknown) { + super(`CultCache store ${filePath} is not readable: ${cause instanceof Error ? cause.message : String(cause)}`, { cause }); + this.name = "StoreUnreadableError"; + this.filePath = filePath; + } +} + /** * A store header as a refusal may show it: echoed in the shape `cultcache.store.v`, * and otherwise described only by its length, since the bytes are the store's. @@ -102,3 +144,10 @@ export function requireV1RecordSlots(record: unknown[]): void { ); } } + +/** Refuses a store array with more or fewer slots than the header, the schema catalog and the records; a header alone is not a store. */ +export function requireStoreSlots(decoded: unknown[]): void { + if (decoded.length !== STORE_SLOTS) { + throw new Error(`CultCache store has ${decoded.length} top-level slots; this runtime reads ${STORE_SLOTS} (header, schema catalog, records).`); + } +} diff --git a/packages/cultcache-ts/test/cult-cache.test.ts b/packages/cultcache-ts/test/cult-cache.test.ts index d767c599..02324273 100644 --- a/packages/cultcache-ts/test/cult-cache.test.ts +++ b/packages/cultcache-ts/test/cult-cache.test.ts @@ -2,7 +2,7 @@ import assert from "node:assert/strict"; import { exec, execFile } from "node:child_process"; import { EventEmitter } from "node:events"; import { existsSync } from "node:fs"; -import { access, lstat, mkdir, mkdtemp, readdir, readFile, symlink, writeFile } from "node:fs/promises"; +import { access, lstat, mkdir, mkdtemp, readdir, readFile, rm, symlink, writeFile } from "node:fs/promises"; import { homedir, tmpdir } from "node:os"; import { dirname, join, resolve, delimiter } from "node:path"; import { test } from "node:test"; @@ -14,9 +14,9 @@ import { z } from "zod"; import { CultCache } from "../src/cult-cache"; import { inspectCultCacheBytes } from "../src/cult-cache-inspector"; import { defineDocumentRegistry, defineDocumentType, documentPayloadBytes, documentPayloadFromBytes } from "../src/document"; -import { SingleFileMessagePackBackingStore } from "../src/single-file-messagepack-backing-store"; -import { refuseUnreadStoreBytes } from "../src/store-format"; -import type { CacheBackingStore, CultCacheEnvelope, CultCacheSchema } from "../src/types"; +import { SingleFileMessagePackBackingStore, readStoreBytes } from "../src/single-file-messagepack-backing-store"; +import { SchemaConflictError, StoreUnreadableError, refuseUnreadStoreBytes } from "../src/store-format"; +import type { AnyCultCacheDocumentDefinition, CacheBackingStore, CultCacheEnvelope, CultCacheSchema } from "../src/types"; const execFileAsync = promisify(execFile); const execAsync = promisify(exec); @@ -461,7 +461,7 @@ test("SingleFileMessagePackBackingStore reads CultCache v1 snapshots by schema i assert.equal(cache.getRequiredEnvelope(noteDocument, "record-1").schemaId, "schema-1"); }); -test("SingleFileMessagePackBackingStore recovers schema-stamped records missing catalog entries", async () => { +test("SingleFileMessagePackBackingStore refuses a record whose schema the catalog does not publish, whatever its payload opens with", async () => { const stampedDocument = defineDocumentType({ type: "runtime-policy", schema: z.tuple([ @@ -501,16 +501,7 @@ test("SingleFileMessagePackBackingStore recovers schema-stamped records missing .withGenericStore(new SingleFileMessagePackBackingStore(storePath)) .build(); - await cache.pullAllBackingStores(); - assert.deepEqual(cache.getRequired(stampedDocument, "record-1"), [ - "tests.schema_stamped_entry.v1", - "schema-stamped", - "still readable", - ]); - assert.equal( - cache.getRequiredEnvelope(stampedDocument, "record-1").schemaId, - "sha256:stale-schema-id-from-cold-record", - ); + await assert.rejects(() => cache.pullAllBackingStores(), StoreUnreadableError); }); test("SingleFileMessagePackBackingStore heals legacy envelopes whose payload was persisted as an object", async () => { @@ -1194,6 +1185,250 @@ test("CultCache v1 MessagePack stores are readable across TS, Rust, C#, and Pyth } }); +// Records and catalog entries a runtime does not own survive its writes. C# writes a store and each other runtime writes its own +// note into it: C# reads its note back, and its record and catalog entry are as C# wrote them. C# then writes again, and the other +// runtime's record and catalog entry are as that runtime wrote them. Identity is compared on the decoded MessagePack values, every +// slot of the entry and of its members included. +test("A runtime's writes lay back another runtime's records and catalog entries unchanged", async () => { + await buildInteropPeers(); + const tempDir = await mkdtemp(join(tmpdir(), "cultcache-lay-back-")); + const csharpWrite = (file: string, runtimeId: string) => + runJsonCommand("csharp-write", dotnetCommand, [csharpInteropDll, "write", "--file", file, "--runtime-id", runtimeId], cultLibRoot); + const others = [ + { + name: "rust", + write: (file: string) => runJsonCommand("rust-write", rustInteropBinary, ["write", "--file", file, "--runtime-id", "rust-writer"], cultcacheRsRoot), + }, + { name: "ts", write: (file: string) => writeTsInteropStore(file, "ts-writer") }, + { + name: "python", + write: (file: string) => runJsonCommand("python-write", pythonCommand, [ + "-m", "cultcache_py.interop", "write", "--file", file, "--runtime-id", "python-writer", + ], cultcachePyRoot, { PYTHONPATH: cultcachePySrc }), + }, + ]; + const stored = async (file: string, key: string) => { + const [, catalog, records] = decode(await readFile(file)) as [string, unknown[][], unknown[][]]; + const record = records.find((candidate) => candidate[0] === key); + assert.ok(record, `${file} holds no record ${key}`); + const entry = catalog.find((candidate) => candidate[0] === record[1]); + assert.ok(entry, `${file} publishes no entry owning ${String(record[1])}`); + return { record, entry }; + }; + + for (const other of others) { + const file = join(tempDir, `${other.name}.cc`); + await csharpWrite(file, "csharp-writer"); + const csharp = await stored(file, "note:csharp-writer"); + + const written = await other.write(file); + assert.deepEqual(await stored(file, "note:csharp-writer"), csharp, `${other.name} rewrote the C# record or its catalog entry`); + const read = await runJsonCommand("csharp-read", dotnetCommand, [csharpInteropDll, "read", "--file", file], cultLibRoot); + assert.ok(read.documentId === "note:csharp-writer" || read.documentId === written.documentId, `C# failed to read after ${other.name}`); + const theirs = await stored(file, written.documentId); + assert.notEqual(theirs.record[1], csharp.record[1], `${other.name} writes under its own id`); + + await csharpWrite(file, "csharp-again"); + assert.deepEqual(await stored(file, written.documentId), theirs, `C# rewrote the ${other.name} record or its catalog entry`); + assert.deepEqual(await stored(file, "note:csharp-writer"), csharp); + } +}); + +// A schema renamed under a stable id: the record's schema id names its type, and the name its catalog entry now carries is metadata. +// A reader that resolved the record by that name would not find its type, so every runtime that can hold the stable id opens it. The +// C# reader is not among them: its schema ids are content hashes, which this file does not carry. +test("CultCache v1 stores stay readable across TS, Rust and Python when a schema is renamed under a stable id", async () => { + await buildInteropPeers(); + const file = join(await mkdtemp(join(tmpdir(), "cultcache-rename-interop-")), "renamed.msgpack"); + const written = await writeTsInteropStore(file, "ts-before-rename"); + await new SingleFileMessagePackBackingStore(file).push({ + key: "note:ts-after-rename", + type: "cultcache.interop-note.renamed", + schemaId: interopNoteDocument.schemaId, + catalogEntry: { + schemaId: interopNoteDocument.schemaId!, + schemaName: "cultcache.interop-note.renamed", + schemaVersion: interopNoteDocument.schemaVersion!, + contentHash: "renamed", + canonicalSchemaJson: interopNoteDocument.canonicalSchemaJson!, + compatibleSchemaIds: [interopNoteDocument.schemaId!], + members: interopNoteDocument.members, + }, + storedAt: new Date().toISOString(), + payload: documentPayloadFromBytes(encode([ + written.schemaVersion, + "note:ts-after-rename", + "ts-after-rename", + "ts wrote a note under the renamed schema", + "The v1 store format is the contract.", + ["interop"], + ])), + }); + const catalog = decode(await readFile(file)) as unknown[][]; + assert.deepEqual((catalog[1] as unknown[][]).map((entry) => [entry[0], entry[1]]), [["cultcache.interop-note", "cultcache.interop-note.renamed"]]); + + const readers = [ + { name: "ts", read: async () => readTsInteropStore(file) }, + { name: "rust", read: async () => runJsonCommand("rust-read", rustInteropBinary, ["read", "--file", file], cultcacheRsRoot) }, + { + name: "python", + read: async () => + runJsonCommand("python-read", pythonCommand, ["-m", "cultcache_py.interop", "read", "--file", file], cultcachePyRoot, { + PYTHONPATH: cultcachePySrc, + }), + }, + ]; + for (const reader of readers) { + const read = await reader.read(); + assert.ok( + ["note:ts-before-rename", "note:ts-after-rename"].includes(read.documentId), + `${reader.name} opened the renamed store and read a note (${read.documentId})`, + ); + assert.equal(read.body, "The v1 store format is the contract."); + } +}); + +// A record TS wrote under an older schema id and another schema name. The C# note type declares that id compatible, so C# opens the +// record, stamps it with its own id when it writes it back, and every runtime opens the result. +test("A C# type that declares a compatible schema id opens a TypeScript record under it and rewrites it under its own id", async () => { + await buildInteropPeers(); + const legacyId = "cultcache.interop-note.legacy-id"; + const file = join(await mkdtemp(join(tmpdir(), "cultcache-compat-interop-")), "legacy.msgpack"); + await new SingleFileMessagePackBackingStore(file).push({ + key: "note:ts-legacy", + type: "cultcache.interop-note.legacy", + schemaId: legacyId, + catalogEntry: { + schemaId: legacyId, + schemaName: "cultcache.interop-note.legacy", + schemaVersion: interopNoteDocument.schemaVersion!, + contentHash: "legacy", + canonicalSchemaJson: interopNoteDocument.canonicalSchemaJson!, + compatibleSchemaIds: [legacyId], + members: interopNoteDocument.members, + }, + storedAt: new Date().toISOString(), + payload: documentPayloadFromBytes(encode([ + "cultcache.interop_note.v1", + "note:ts-legacy", + "ts-legacy", + "ts wrote a note under an older schema id", + "The v1 store format is the contract.", + ["interop"], + ])), + }); + const before = decode(await readFile(file)) as unknown[][]; + assert.equal((before[2] as unknown[][])[0]![1], legacyId); + + const rewritten = await runJsonCommand("csharp-rewrite", dotnetCommand, [csharpInteropDll, "rewrite", "--file", file], cultLibRoot); + assert.equal(rewritten.documentId, "note:ts-legacy"); + const after = decode(await readFile(file)) as unknown[][]; + const stampedId = (after[2] as unknown[][])[0]![1] as string; + assert.notEqual(stampedId, legacyId, "the rewrite stamped the record with the registered id"); + const entry = (after[1] as unknown[][]).find((candidate) => candidate[0] === stampedId) as unknown[]; + assert.ok(entry, "the catalog publishes the id the record now carries"); + assert.ok((entry[5] as string[]).includes(legacyId), "the registered entry still lists the older id"); + + const readers = [ + { name: "csharp", read: async () => runJsonCommand("csharp-read", dotnetCommand, [csharpInteropDll, "read", "--file", file], cultLibRoot) }, + { name: "ts", read: async () => readTsInteropStore(file) }, + { name: "rust", read: async () => runJsonCommand("rust-read", rustInteropBinary, ["read", "--file", file], cultcacheRsRoot) }, + { + name: "python", + read: async () => + runJsonCommand("python-read", pythonCommand, ["-m", "cultcache_py.interop", "read", "--file", file], cultcachePyRoot, { + PYTHONPATH: cultcachePySrc, + }), + }, + ]; + for (const reader of readers) { + const read = await reader.read(); + assert.equal(read.documentId, "note:ts-legacy", `${reader.name} opened the rewritten store`); + assert.equal(read.body, "The v1 store format is the contract."); + } +}); + +test("CultCache element ids cross C#, TypeScript, Python and Rust as ordinary members", async () => { + await buildInteropPeers(); + const tempDir = await mkdtemp(join(tmpdir(), "cultcache-deck-")); + const file = join(tempDir, "deck.cc"); + const written = await runJsonCommand("csharp-write-deck", dotnetCommand, [ + csharpInteropDll, "write-deck", "--file", file, + ], cultLibRoot); + assert.equal(written.ids.length, 3); + assert.equal(new Set(written.ids).size, 3, "two look-alike marks got distinct ids"); + for (const id of written.ids) assert.match(id, /^[0-9a-f]{12}$/u); + + // TypeScript: the id is slot 1 of each mark, read from the raw payload. + const store = decode(await readFile(file)) as any[]; + const payload = decode(store[2][0][3]) as any[]; + assert.deepEqual(payload[1].map((mark: any[]) => mark[1]), written.ids); + + // Python and Rust read the same store. + const python = await runJsonCommand("python-deck", pythonCommand, [ + "-c", + "import msgpack,json,sys;s=msgpack.unpackb(open(sys.argv[1],'rb').read(),raw=False);p=msgpack.unpackb(s[2][0][3],raw=False);print(json.dumps({'ids':[m[1] for m in p[1]]}))", + file, + ], cultcachePyRoot); + assert.deepEqual(python.ids, written.ids); + const rust = await runJsonCommand("rust-read-deck", rustInteropBinary, ["read-deck", "--file", file], cultcacheRsRoot); + assert.deepEqual(rust.ids, written.ids); + + // TypeScript writes the ids; C# keeps them. + const tsIds = ["a00000000001", "a00000000002", "a00000000003"]; + store[2][0][3] = encode(["deck:csharp", [["twin", tsIds[0]], ["twin", tsIds[1]], ["other", tsIds[2]]]]); + const tsFile = join(tempDir, "ts-deck.cc"); + await writeFile(tsFile, encode(store)); + const back = await runJsonCommand("csharp-read-deck", dotnetCommand, [ + csharpInteropDll, "read-deck", "--file", tsFile, + ], cultLibRoot); + assert.deepEqual(back.ids, tsIds); +}); + +// A C# record edited in place is written as edited, however the consumer reached it: a decoded snapshot record's payload bytes, a +// variant override's value bytes, a collection element or a nested object of the document the cache holds. Rust and Python read the +// edit through their own readers; the variant and the nested object are read by decoding the file with Python's msgpack. +test("C# writes an edit made in place, and Rust and Python read the new value", async () => { + await buildInteropPeers(); + const tempDir = await mkdtemp(join(tmpdir(), "cultcache-edit-in-place-")); + const csharp = (name: string, args: string[]) => runJsonCommand(name, dotnetCommand, [csharpInteropDll, ...args], cultLibRoot); + const edit = (file: string, kind: string) => csharp(`csharp-edit-${kind}`, ["edit-in-place", "--file", file, "--kind", kind]); + const readers = (file: string) => [ + runJsonCommand("rust-read", rustInteropBinary, ["read", "--file", file], cultcacheRsRoot), + runJsonCommand("python-read", pythonCommand, ["-m", "cultcache_py.interop", "read", "--file", file], cultcachePyRoot, { PYTHONPATH: cultcachePySrc }), + ]; + const dump = (file: string) => runJsonCommand("python-dump", pythonCommand, [ + "-c", + "import msgpack,json,sys\n" + + "s=msgpack.unpackb(open(sys.argv[1],'rb').read(),raw=False)\n" + + "print(json.dumps([{'key':r[0],'overrides':[o[3] for o in r[4][1]]} if len(r)>4 else {'key':r[0],'payload':msgpack.unpackb(r[3],raw=False)} for r in s[2]]))", + file, + ], cultcachePyRoot); + + const payload = join(tempDir, "payload.cc"); + await csharp("csharp-write", ["write", "--file", payload, "--runtime-id", "csharp-writer"]); + await edit(payload, "payload"); + for (const read of await Promise.all(readers(payload))) assert.equal(read.title, "csharp-writer WROTE a CultCache note"); + + const collection = join(tempDir, "collection.cc"); + await csharp("csharp-write", ["write", "--file", collection, "--runtime-id", "csharp-writer"]); + await edit(collection, "collection"); + for (const read of await Promise.all(readers(collection))) assert.deepEqual(read.tags, ["EDITED", "csharp", "interop"]); + + const override = join(tempDir, "override.cc"); + await csharp("csharp-write", ["write", "--file", override, "--runtime-id", "csharp-writer"]); + await edit(override, "override"); + const variant = (await dump(override)).find((record: any) => record.key === "note:variant"); + assert.deepEqual(variant.overrides, ["note:variant", "VARIANT title"]); + + const object = join(tempDir, "object.cc"); + await csharp("csharp-write-deck", ["write-deck", "--file", object]); + await edit(object, "object"); + const deck = (await dump(object)).find((record: any) => record.key === "deck:csharp"); + assert.equal(deck.payload[1][0][0], "EDITED"); + assert.equal(deck.payload[1][1][0], "twin", "the neighbouring mark is untouched"); +}); + test("CultCache interop reader accepts missing compatible trailing slots and rejects mismatched slots", async () => { const tempDir = await mkdtemp(join(tmpdir(), "cultcache-interop-")); const compatible = join(tempDir, "compatible.msgpack"); @@ -1236,40 +1471,386 @@ test("CultCache inspector decodes v1 store catalog and record payloads from byte ]); }); -test("CultCache inspector recovers schema-stamped records missing catalog entries", async () => { +test("CultCache inspector refuses a record whose schema the catalog does not publish, and a store of the wrong slot count", async () => { const tempDir = await mkdtemp(join(tmpdir(), "cultcache-inspector-")); - const file = join(tempDir, "missing-catalog.cc"); + const record = [ + "record-1", + "sha256:stale-schema-id-from-cold-record", + "2026-06-25T12:00:00Z", + encode(["tests.schema_stamped_entry.v1", "schema-stamped", "still readable"]), + ]; + assert.throws( + () => inspectCultCacheBytes("missing-catalog.cc", encode(["cultcache.store.v1", [], [record]])), + /references missing schema id/u, + ); + assert.throws(() => inspectCultCacheBytes("header-only.cc", encode(["cultcache.store.v1"])), /top-level slots/u); + assert.throws( + () => inspectCultCacheBytes("extra-slot.cc", encode(["cultcache.store.v1", [], [], 0])), + /top-level slots/u, + ); + await rm(tempDir, { recursive: true, force: true }); +}); + +// A record written under an id its schema only lists as compatible is stamped with the registered id, and the store the write +// leaves opens and takes the next write. The envelope carries a catalog entry under the foreign id, as a replicated one does. +const foreignIdDocument = defineDocumentType({ + type: "tests.foreign-id", + schema: z.object({ name: z.string() }), + schemaId: "tests.foreign-id.current", + schemaName: "tests.foreign-id", + schemaVersion: "tests.foreign_id.v1", + compatibleSchemaIds: ["tests.foreign-id.current", "tests.foreign-id.older"], +}); + +async function reopenForeignIdStore(file: string): Promise { + const cache = CultCache.builder() + .withDocumentType(foreignIdDocument) + .withGenericStore(new SingleFileMessagePackBackingStore(file)) + .build(); + await cache.pullAllBackingStores(); + return cache; +} + +function schemaIdsOf(bytes: Uint8Array): { record: string; catalog: string[] } { + const decoded = decode(bytes) as [string, unknown[][], unknown[][]]; + return { record: decoded[2][0]![1] as string, catalog: decoded[1].map((entry) => entry[0] as string) }; +} + +test("putEnvelope under a compatible schema id writes a store that reopens and takes the next write", async () => { + const file = join(await mkdtemp(join(tmpdir(), "cultcache-foreign-id-")), "store.msgpack"); + const cache = await reopenForeignIdStore(file); + const seed = await cache.put(foreignIdDocument, "seed", { name: "seed" }); + const payload = cache.getRequiredEnvelope(foreignIdDocument, "seed").payload; + + await cache.putEnvelope(foreignIdDocument, { + key: "foreign", + type: "tests.foreign-id", + schemaId: "tests.foreign-id.older", + payload, + storedAt: "2026-09-30T00:00:00.0000000Z", + catalogEntry: { + schemaId: "tests.foreign-id.older", + schemaName: "tests.foreign-id", + schemaVersion: "tests.foreign_id.v1", + contentHash: "tests.foreign-id.older", + canonicalSchemaJson: "", + compatibleSchemaIds: ["tests.foreign-id.older"], + members: [], + }, + }); + + const ids = schemaIdsOf(await readFile(file)); + assert.deepEqual(ids, { record: "tests.foreign-id.current", catalog: ["tests.foreign-id.current"] }); + const reopened = await reopenForeignIdStore(file); + assert.deepEqual(reopened.getRequired(foreignIdDocument, "foreign"), seed); + await reopened.put(foreignIdDocument, "next", { name: "next" }); + assert.deepEqual((await reopenForeignIdStore(file)).getRequired(foreignIdDocument, "next"), { name: "next" }); +}); + +test("the store publishes a record's schema id even when the envelope's catalog entry does not", async () => { + const file = join(await mkdtemp(join(tmpdir(), "cultcache-store-publishes-")), "store.msgpack"); + const store = new SingleFileMessagePackBackingStore(file); + await store.push({ + key: "k", + type: "tests.store-publishes", + schemaId: "tests.store-publishes.record", + payload: documentPayloadFromBytes(encode({ name: "k" })), + storedAt: "2026-09-30T00:00:00.0000000Z", + catalogEntry: { + schemaId: "tests.store-publishes.other", + schemaName: "tests.store-publishes", + schemaVersion: "tests.store_publishes.v1", + contentHash: "tests.store-publishes.other", + canonicalSchemaJson: "", + compatibleSchemaIds: ["tests.store-publishes.other"], + members: [], + }, + }); + assert.deepEqual(schemaIdsOf(await readFile(file)), { record: "tests.store-publishes.record", catalog: ["tests.store-publishes.record"] }); + assert.equal((await new SingleFileMessagePackBackingStore(file).pullAll()).length, 1); +}); + +// The catalog a write leaves is derived from its records: one entry per carried id, chosen and never merged, the same whatever order +// the records arrive in. A registered (supplied) entry wins over an arrived (read back) entry with the same own id. +const writerEntry = (schemaId: string, schemaName: string, contentHash: string, compatibleSchemaIds: string[]) => ({ + schemaId, + schemaName, + schemaVersion: `${schemaName}.v1`, + contentHash, + canonicalSchemaJson: "", + compatibleSchemaIds, + members: [], +}); +const writerRecord = (key: string, type: string, schemaId: string, catalogEntry?: ReturnType) => ({ + key, + type, + schemaId, + payload: documentPayloadFromBytes(encode({ key })), + storedAt: "2026-09-30T00:00:00.0000000Z", + ...(catalogEntry ? { catalogEntry } : {}), +}); +const rawCatalogEntry = (entry: ReturnType) => [ + entry.schemaId, + entry.schemaName, + entry.schemaVersion, + entry.contentHash, + entry.canonicalSchemaJson, + entry.compatibleSchemaIds, + [], +]; +const catalogOf = async (file: string): Promise => + (decode(await readFile(file)) as [string, unknown[][], unknown[][]])[1]; + +test("a registered entry is written as it is, over the arrived entries with the same own id, in either order", async () => { + const dir = await mkdtemp(join(tmpdir(), "cultcache-registered-wins-")); + const stale1 = writerEntry("tests.x", "tests.n", "a-stale-1", ["tests.x", "old"]); + const stale2 = writerEntry("tests.x", "tests.n", "a-stale-2", ["tests.x"]); + const registered = writerRecord("a", "tests.n", "tests.x", writerEntry("tests.x", "tests.n", "fresh", ["tests.x"])); + + for (const [name, entries] of [["12", [stale1, stale2]], ["21", [stale2, stale1]]] as const) { + const file = join(dir, `${name}.msgpack`); + await writeFile( + file, + encode(["cultcache.store.v1", entries.map(rawCatalogEntry), [["b", "tests.x", "2026-09-30T00:00:00.0000000Z", encode({ b: 1 })]]]), + ); + await new SingleFileMessagePackBackingStore(file).push(registered); + const catalog = await catalogOf(file); + assert.equal(catalog.length, 1, name); + assert.equal(catalog[0]![3], "fresh", `${name}: the registered entry's hash is written`); + assert.deepEqual(catalog[0]![5], ["tests.x"], `${name}: written as chosen, no union with the arrived entries' ids`); + } +}); + +test("a rename with a stable schema id: a whole-view write names the registered schema", async () => { + const dir = await mkdtemp(join(tmpdir(), "cultcache-rename-")); + const file = join(dir, "rename.msgpack"); + await writeFile( + file, + encode(["cultcache.store.v1", [rawCatalogEntry(writerEntry("tests.x", "tests.old", "stale", ["tests.x"]))], [["a", "tests.x", "2026-09-30T00:00:00.0000000Z", encode({ a: 1 })]]]), + ); + await new SingleFileMessagePackBackingStore(file).pushAll([writerRecord("a", "tests.new", "tests.x", writerEntry("tests.x", "tests.new", "fresh", ["tests.x"]))]); + assert.deepEqual((await catalogOf(file)).map((entry) => [entry[1], entry[3]]), [["tests.new", "fresh"]]); +}); + +test("two entries of one tier that share an own id and disagree on the schema name refuse the write, typed", async () => { + const dir = await mkdtemp(join(tmpdir(), "cultcache-disagree-")); + const first = writerRecord("b", "tests.first", "tests.x", writerEntry("tests.x", "tests.first", "h1", ["tests.x"])); + const second = writerRecord("a", "tests.first", "tests.x", writerEntry("tests.x", "tests.second", "h2", ["tests.x"])); + for (const [name, order] of [["fs", [first, second]], ["sf", [second, first]]] as const) { + const file = join(dir, `${name}.msgpack`); + await assert.rejects( + () => new SingleFileMessagePackBackingStore(file).pushAll([...order]), + (error) => + error instanceof SchemaConflictError && + error.schemaId === "tests.x" && + error.recordKey === "a" && + [...error.schemaNames].sort().join() === "tests.first,tests.second", + name, + ); + } +}); + +test("records of different types under one schema id refuse the write, typed, and no record changes type", async () => { + const dir = await mkdtemp(join(tmpdir(), "cultcache-retype-")); + const a = writerRecord("a", "tests.n", "tests.y", writerEntry("tests.x", "tests.n", "h1", ["tests.x", "tests.y"])); + const b = writerRecord("b", "tests.m", "tests.y", writerEntry("tests.y", "tests.m", "h2", ["tests.y"])); + + for (const [name, order] of [["ab", [a, b]], ["ba", [b, a]]] as const) { + const file = join(dir, `${name}.msgpack`); + await assert.rejects( + () => new SingleFileMessagePackBackingStore(file).pushAll([...order]), + (error) => error instanceof SchemaConflictError && error.schemaId === "tests.y" && error.schemaNames.join() === "tests.m,tests.n", + name, + ); + } +}); + +test("an entry that owns an id is chosen over one that lists it, and an entry no record needs is not written", async () => { + const dir = await mkdtemp(join(tmpdir(), "cultcache-own-first-")); + const owner = writerRecord("a", "tests.n", "tests.y", writerEntry("tests.y", "tests.n", "h1", ["tests.y"])); + const lister = writerRecord("b", "tests.n", "tests.x", writerEntry("tests.x", "tests.n", "h2", ["tests.x", "tests.y"])); + const onlyOwner = join(dir, "own.msgpack"); + await new SingleFileMessagePackBackingStore(onlyOwner).pushAll([owner]); + assert.deepEqual((await catalogOf(onlyOwner)).map((entry) => entry[0]), ["tests.y"]); + + // The lister carried only its own id: its list of y is not needed, and the owner's entry publishes y. + for (const [name, order] of [["lo", [lister, owner]], ["ol", [owner, lister]]] as const) { + const file = join(dir, `${name}.msgpack`); + await new SingleFileMessagePackBackingStore(file).pushAll([...order]); + const catalog = await catalogOf(file); + assert.deepEqual(catalog.map((entry) => [entry[0], entry[5]]).sort(), [["tests.x", ["tests.x", "tests.y"]], ["tests.y", ["tests.y"]]], name); + } +}); + +test("entries of one tier that tie are taken in one fixed order, and a registered entry that lists an id is chosen over an arrived one", async () => { + const dir = await mkdtemp(join(tmpdir(), "cultcache-tie-")); + // The content hash orders them against the order their compatible ids would give: h1 sorts first, its list sorts last. + const one = writerRecord("a", "tests.n", "tests.x", writerEntry("tests.x", "tests.n", "h2", ["tests.x"])); + const two = writerRecord("b", "tests.n", "tests.x", writerEntry("tests.x", "tests.n", "h1", ["tests.x", "tests.y"])); + for (const [name, order] of [["12", [one, two]], ["21", [two, one]]] as const) { + const file = join(dir, `${name}.msgpack`); + await new SingleFileMessagePackBackingStore(file).pushAll([...order]); + assert.deepEqual((await catalogOf(file)).map((entry) => entry[3]), ["h1"], name); + } + + // z sits under old, which the arrived entry (id x) lists; the supplied entry (id r) lists it too: the registered one publishes it. + const file = join(dir, "lister.msgpack"); await writeFile( file, encode([ "cultcache.store.v1", - [], - [ - [ - "record-1", - "sha256:stale-schema-id-from-cold-record", - "2026-06-25T12:00:00Z", - encode([ - "tests.schema_stamped_entry.v1", - "schema-stamped", - "still readable", - ]), - ], - ], + [rawCatalogEntry(writerEntry("tests.x", "tests.n", "stale", ["tests.x", "old"]))], + [["z", "old", "2026-09-30T00:00:00.0000000Z", encode({ z: 1 })]], + ]), + ); + await new SingleFileMessagePackBackingStore(file).push(writerRecord("a", "tests.n", "tests.r", writerEntry("tests.r", "tests.n", "fresh", ["tests.r", "old"]))); + assert.deepEqual((await catalogOf(file)).map((entry) => entry[0]), ["tests.r"]); +}); + +// A registered entry that owns an id keeps it against an arrived entry that owns it too and lists the id another record sits under, +// whichever of the two ids sorts first: the arrived entry's list is not merged in, so the record under the listed id is published by +// no chosen entry. +for (const [ownId, listedId] of [["tests.a", "tests.b"], ["tests.b", "tests.a"]] as const) { + test(`an arrived entry that owns a registered id does not replace it nor publish what it lists (${ownId})`, async () => { + const dir = await mkdtemp(join(tmpdir(), "cultcache-guard-")); + const file = join(dir, "guard.msgpack"); + await writeFile( + file, + encode([ + "cultcache.store.v1", + [rawCatalogEntry(writerEntry(ownId, "tests.arrived", "stale", [ownId, listedId]))], + [["k2", listedId, "2026-09-30T00:00:00.0000000Z", encode({ k: 2 })]], + ]), + ); + const before = await readFile(file); + await assert.rejects( + () => new SingleFileMessagePackBackingStore(file).push(writerRecord("k1", "tests.registered", ownId, writerEntry(ownId, "tests.registered", "fresh", [ownId]))), + (error) => error instanceof SchemaConflictError && error.schemaId === listedId && error.recordKey === "k2", + ); + assert.ok(before.equals(await readFile(file)), "the file is left as it was"); + }); +} + +// Ids are taken in sorted order, so which entry survives, and so which id the refusal names, does not depend on the order the records +// arrive in. +test("the refusal names the same id whatever order the records arrive in", async () => { + const dir = await mkdtemp(join(tmpdir(), "cultcache-sorted-")); + const one = writerRecord("r1", "tests.n", "tests.y", writerEntry("tests.x", "tests.n", "h1", ["tests.x", "tests.y"])); + const two = writerRecord("r2", "tests.n", "tests.z", writerEntry("tests.x", "tests.n", "h2", ["tests.x", "tests.z"])); + for (const [name, order] of [["yz", [one, two]], ["zy", [two, one]]] as const) { + await assert.rejects( + () => new SingleFileMessagePackBackingStore(join(dir, `${name}.msgpack`)).pushAll([...order]), + (error) => error instanceof SchemaConflictError && error.schemaId === "tests.y" && error.recordKey === "r1", + name, + ); + } +}); + +// A write may not retype records already in the file: across tiers the id keeps the type its records resolve to, unless the entries of +// both tiers own it, which is a rename under a stable id and the registered descriptor wins. +test("a write that would change the type an existing record resolves to refuses, typed, and leaves the file", async () => { + const dir = await mkdtemp(join(tmpdir(), "cultcache-cross-tier-")); + const at = "2026-09-30T00:00:00.0000000Z"; + const cases = [ + { + name: "registered entry lists the id, arrived entry owns it", + catalog: writerEntry("tests.x", "tests.arrived", "h1", ["tests.x"]), + registered: writerRecord("a", "tests.registered", "tests.x", writerEntry("tests.r", "tests.registered", "h2", ["tests.r", "tests.x"])), + }, + { + name: "registered entry owns the id, arrived entry lists it", + catalog: writerEntry("tests.r", "tests.arrived", "h1", ["tests.r", "tests.x"]), + registered: writerRecord("a", "tests.registered", "tests.x", writerEntry("tests.x", "tests.registered", "h2", ["tests.x"])), + }, + ]; + for (const { name, catalog, registered } of cases) { + const file = join(dir, `${name}.msgpack`); + await writeFile(file, encode(["cultcache.store.v1", [rawCatalogEntry(catalog)], [["z", "tests.x", at, encode({ z: 1 })]]])); + const before = await readFile(file); + await assert.rejects( + () => new SingleFileMessagePackBackingStore(file).push(registered), + (error) => + error instanceof SchemaConflictError && + error.schemaId === "tests.x" && + [...error.schemaNames].sort().join() === "tests.arrived,tests.registered", + name, + ); + assert.ok(before.equals(await readFile(file)), `${name}: the file is left as it was`); + } + + // The same registered record renames the schema when both tiers own the id: it writes, and the reader finds every record. + const file = join(dir, "rename.msgpack"); + await writeFile(file, encode(["cultcache.store.v1", [rawCatalogEntry(writerEntry("tests.x", "tests.arrived", "h1", ["tests.x"]))], [["z", "tests.x", at, encode({ z: 1 })]]])); + await new SingleFileMessagePackBackingStore(file).push(writerRecord("a", "tests.registered", "tests.x", writerEntry("tests.x", "tests.registered", "h2", ["tests.x"]))); + assert.deepEqual((await catalogOf(file)).map((entry) => [entry[0], entry[1]]), [["tests.x", "tests.registered"]]); + assert.equal((await new SingleFileMessagePackBackingStore(file).pullAll()).length, 2); +}); + +test("a record no chosen entry publishes refuses the write, typed, and leaves the file", async () => { + const dir = await mkdtemp(join(tmpdir(), "cultcache-orphan-")); + const file = join(dir, "orphan.msgpack"); + // z sits under old, which only the arrived entry (id x) lists; the supplied entry owns x and does not. + await writeFile( + file, + encode([ + "cultcache.store.v1", + [rawCatalogEntry(writerEntry("tests.x", "tests.legacy", "stale", ["tests.x", "old"]))], + [["z", "old", "2026-09-30T00:00:00.0000000Z", encode({ z: 1 })]], + ]), + ); + const before = await readFile(file); + await assert.rejects( + () => new SingleFileMessagePackBackingStore(file).push(writerRecord("a", "tests.legacy", "tests.x", writerEntry("tests.x", "tests.legacy", "fresh", ["tests.x"]))), + (error) => error instanceof SchemaConflictError && error.schemaId === "old" && error.recordKey === "z", + ); + assert.ok(before.equals(await readFile(file)), "the file is left as it was"); +}); + +test("an id an entry owns names that entry, not one that lists it as compatible", async () => { + const dir = await mkdtemp(join(tmpdir(), "cultcache-own-id-")); + for (const vector of ["own-id-over-compatible-v3.bin", "compatible-before-owner-v3.bin"]) { + const file = await copyVector(dir, join(c2aVectors, "readability", vector), "own.msgpack"); + const envelopes = await new SingleFileMessagePackBackingStore(file).pullAll(); + assert.deepEqual(envelopes.map((envelope) => envelope.type), ["vectors.item", "vectors.item"], vector); + const inspection = inspectCultCacheBytes("own.msgpack", await readFile(file)); + assert.deepEqual(inspection.records.map((record) => record.schemaName), ["vectors.item", "vectors.item"], vector); + } +}); + +// The inspector reads what the reader reads: a record under an id its catalog lists only as compatible is inspected. +test("CultCache inspector reads a record whose schema the catalog publishes only as a compatible id", async () => { + const bytes = await readFile(join(c2aVectors, "readability", "compatible-id-only-v3.bin")); + const inspection = inspectCultCacheBytes("compatible.msgpack", bytes); + assert.deepEqual(inspection.records.map((record) => record.schemaName), ["vectors.item", "vectors.item"]); +}); + +test("a record loaded under a compatible schema id is written back under the id its catalog entry carries", async () => { + const file = join(await mkdtemp(join(tmpdir(), "cultcache-foreign-id-")), "store.msgpack"); + const payload = encode({ name: "old" }); + await writeFile( + file, + encode([ + "cultcache.store.v1", + [[ + "tests.foreign-id.current", + "tests.foreign-id", + "tests.foreign_id.v1", + "tests.foreign-id.current", + "", + ["tests.foreign-id.current", "tests.foreign-id.older"], + [], + ]], + [["old", "tests.foreign-id.older", "2026-09-30T00:00:00.0000000Z", payload]], ]), ); - const inspection = inspectCultCacheBytes(file, await readFile(file)); - assert.equal(inspection.catalog.length, 1); - assert.equal(inspection.catalog[0]?.schemaId, "sha256:stale-schema-id-from-cold-record"); - assert.equal(inspection.catalog[0]?.schemaName, "tests.schema_stamped_entry"); - assert.equal(inspection.records[0]?.schemaName, "tests.schema_stamped_entry"); - assert.deepEqual(inspection.records[0]?.payloadPreview, [ - "tests.schema_stamped_entry.v1", - "schema-stamped", - "still readable", - ]); + const cache = await reopenForeignIdStore(file); + assert.equal(cache.getRequiredEnvelope(foreignIdDocument, "old").schemaId, "tests.foreign-id.current"); + await cache.put(foreignIdDocument, "next", { name: "next" }); + const reopened = await reopenForeignIdStore(file); + assert.deepEqual(reopened.getRequired(foreignIdDocument, "old"), { name: "old" }); + assert.deepEqual(reopened.getRequired(foreignIdDocument, "next"), { name: "next" }); }); interface InteropNote { @@ -1482,20 +2063,24 @@ test("SingleFileMessagePackBackingStore refusals say what was found without blam ); }); -test("pushAll refuses a store in a format this runtime does not read and leaves it byte-identical", async () => { - const vector = await readFile(resolve(cultLibRoot, "tests", "vectors", "document-variants-c2a", "v3-base.msgpack")); - const envelope = { key: "probe", type: "probe", payload: documentPayloadFromBytes(new Uint8Array([0x80])), storedAt: "2026-10-22T00:00:00Z" }; - for (const bytes of [vector, vector.subarray(0, Math.floor(vector.length / 2))]) { - const file = join(await mkdtemp(join(tmpdir(), "cultcache-pushall-v3-")), "store.msgpack"); - await writeFile(file, bytes); - await assert.rejects(() => new SingleFileMessagePackBackingStore(file).pushAll([envelope]), /cultcache\.store\.v3/u); - assert.equal(Buffer.compare(await readFile(file), bytes), 0); +test("SingleFileMessagePackBackingStore refusal says what was found and names every format it reads", async () => { + const dir = await mkdtemp(join(tmpdir(), "cultcache-refusal-")); + const file = join(dir, "store.cc"); + const reads = 'it reads "cultcache.store.v1" and "cultcache.store.v3" only.'; + for (const [header, found] of [ + ["cultcache.store.v12", '"cultcache.store.v12"'], + ["cultcache.store.SECRET", "an unrecognised cultcache.store.* header of 22 bytes"], + ["cultcache.store.v", "an unrecognised cultcache.store.* header of 17 bytes"], + // Digits are ASCII only: two Arabic-Indic digits are four bytes and are not the known shape. + ["cultcache.store.v١٢", "an unrecognised cultcache.store.* header of 21 bytes"], + ] as const) { + await writeFile(file, encode([header, [], []])); + await assert.rejects( + () => new SingleFileMessagePackBackingStore(file).pullAll(), + (error: Error) => error.message.includes(`CultCache store format ${found} is not one this runtime reads; ${reads}`), + header, + ); } - const healed = join(await mkdtemp(join(tmpdir(), "cultcache-pushall-heal-")), "store.msgpack"); - await writeFile(healed, new Uint8Array([0x93, 0xa0])); - const store = new SingleFileMessagePackBackingStore(healed); - await store.pushAll([envelope]); - assert.equal((await store.pullAll()).length, 1); }); function headerRefused(bytes: Uint8Array): boolean { @@ -1523,42 +2108,37 @@ test("a store's header is judged from its first bytes, in every MessagePack arra const leads = [[0x91], [0x9f], [0xdc, 0x00, 0x01], [0xdc, 0x01, 0x00], [0xdd, 0x00, 0x00, 0x00, 0x01], [0xdd, 0x00, 0x01, 0x00, 0x00]]; for (const lead of leads) { for (const form of ["fix", "8", "16", "32"] as const) { - const v3 = storeStart(lead, "cultcache.store.v3", form); - assert.equal(headerRefused(v3), true, `${lead} ${form} v3`); - assert.equal(headerRefused(v3.subarray(0, v3.length - 1)), false, `${lead} ${form} v3 one byte short`); + const v4 = storeStart(lead, "cultcache.store.v4", form); + assert.equal(headerRefused(v4), true, `${lead} ${form} v4`); + assert.equal(headerRefused(v4.subarray(0, v4.length - 1)), false, `${lead} ${form} v4 one byte short`); assert.equal(headerRefused(storeStart(lead, "cultcache.store.v1", form)), false, `${lead} ${form} v1`); - assert.equal(headerRefused(storeStart(lead, "other.store.v3", form)), false, `${lead} ${form} other`); + assert.equal(headerRefused(storeStart(lead, "cultcache.store.v3", form)), false, `${lead} ${form} v3`); + assert.equal(headerRefused(storeStart(lead, "other.store.v4", form)), false, `${lead} ${form} other`); assert.equal(headerRefused(storeStart(lead, "", form)), false, `${lead} ${form} empty`); } } // The longest fixstr, and a string long enough to need str16. - assert.equal(headerRefused(storeStart([0x91], `cultcache.store.v3${"x".repeat(13)}`, "fix")), true); - assert.equal(headerRefused(storeStart([0x91], `cultcache.store.v3${"x".repeat(300)}`, "16")), true); - assert.equal(headerRefused(storeStart([0x91], `cultcache.store.v3${"x".repeat(300)}`, "32")), true); - // A view into a larger buffer is read from its own start. - const padded = new Uint8Array([9, 9, ...storeStart([0x91], "cultcache.store.v3", "fix")]); - assert.equal(headerRefused(padded.subarray(2)), true); -}); - -test("a stamp followed by bytes that are not UTF-8 gets the verdict a read gives", async () => { - const stamp = [...Buffer.from("cultcache.store.v3", "utf8")]; - const forms = { fixstr: [0x91, 0xb3, ...stamp, 0xff], str8: [0x91, 0xd9, 19, ...stamp, 0xff] }; - const envelope = { key: "probe", type: "probe", payload: documentPayloadFromBytes(new Uint8Array([0x80])), storedAt: "2026-10-22T00:00:00Z" }; - for (const [name, bytes] of Object.entries(forms)) { - const file = join(await mkdtemp(join(tmpdir(), "cultcache-stamp-")), "store.msgpack"); - await writeFile(file, new Uint8Array(bytes)); - const store = new SingleFileMessagePackBackingStore(file); - await assert.rejects(() => store.pullAll(), /not one this runtime reads/u, `${name} read`); - await assert.rejects(() => store.pushAll([envelope]), /not one this runtime reads/u, `${name} pushAll`); - assert.deepEqual([...(await readFile(file))], bytes, `${name} left as it was`); + assert.equal(headerRefused(storeStart([0x91], `cultcache.store.v4${"x".repeat(13)}`, "fix")), true); + assert.equal(headerRefused(storeStart([0x91], `cultcache.store.v4${"x".repeat(300)}`, "16")), true); + assert.equal(headerRefused(storeStart([0x91], `cultcache.store.v4${"x".repeat(300)}`, "32")), true); + // A view into a larger buffer is read from its own start, in every form that reads a length through the DataView. + for (const lead of leads) { + for (const form of ["fix", "8", "16", "32"] as const) { + const padded = new Uint8Array([7, 7, 7, ...storeStart(lead, "cultcache.store.v4", form)]); + assert.equal(padded.subarray(3).byteOffset, 3); + assert.equal(headerRefused(padded.subarray(3)), true, `${lead} ${form} v4 at a byte offset`); + } } + // The stamp is judged as written: a stamp with a space is not the stamp this runtime reads, and not one it ignores. + assert.equal(headerRefused(storeStart([0x91], "cultcache.store.v3 ", "fix")), true, "a v3 stamp with a trailing space is a header this runtime does not read"); + assert.equal(headerRefused(storeStart([0x91], " cultcache.store.v4", "fix")), false, "a leading space makes it no cultcache.store.* header"); }); test("bytes with no readable store stamp are not refused", () => { - const stamp = [...Buffer.from("cultcache.store.v3", "utf8")]; + const stamp = [...Buffer.from("cultcache.store.v4", "utf8")]; const cases: Record = { empty: [], - "empty array": [0x90], + "empty array": [0x90, 0xb2, ...stamp], "empty array16": [0xdc, 0x00, 0x00, 0xb2, ...stamp], "empty array32": [0xdd, 0x00, 0x00, 0x00, 0x00, 0xb2, ...stamp], "array16 cut before its count": [0xdc, 0x00], @@ -1567,9 +2147,6 @@ test("bytes with no readable store stamp are not refused", () => { "first element is not a string": [0x92, 0x01, 0xb2, ...stamp], "a bare string": [0xb2, ...stamp], "a map": [0x81, 0xb2, ...stamp], - "a map whose bytes read as an array32 count": [0x81, 0x00, 0x00, 0x00, 0x01, 0xb2, ...stamp], - "a small integer before a stamp-looking body": [0x91, 0x12, ...stamp], - "a uint32 before a stamp-looking body": [0x91, 0xce, 0x00, 0x00, 0x00, 0x12, ...stamp], "str8 cut before its length": [0x91, 0xd9], "str16 cut before its length": [0x91, 0xda, 0x00], "str32 cut before its length": [0x91, 0xdb, 0x00, 0x00, 0x00], @@ -1579,13 +2156,51 @@ test("bytes with no readable store stamp are not refused", () => { } }); -test("pushAll leaves a store in a newer format alone when soft, and replaces a store it reads", async () => { +test("a newer store is refused by name whatever its body: truncated, undecodable or trailing, by a read, a whole write and the inspector", async () => { + const envelope = { key: "probe", type: "probe", payload: documentPayloadFromBytes(new Uint8Array([0x80])), storedAt: "2026-10-22T00:00:00Z" }; + const whole = encode(["cultcache.store.v4", [], [["k", "s", "t", new Uint8Array([0x90])]]]); + const bodies: Record = { + truncated: whole.subarray(0, whole.length - 3), + "header only": whole.subarray(0, 20), + undecodable: Uint8Array.of(...whole.subarray(0, 20), 0xc1), + trailing: Uint8Array.of(...whole, 0xff), + whole, + }; + for (const [name, bytes] of Object.entries(bodies)) { + const file = join(await mkdtemp(join(tmpdir(), "cultcache-newer-")), "store.msgpack"); + await writeFile(file, bytes); + const store = new SingleFileMessagePackBackingStore(file); + await assert.rejects(() => store.pullAll(), /not one this runtime reads/u, `${name} read`); + await assert.rejects(() => store.pushAll([envelope]), /not one this runtime reads/u, `${name} pushAll`); + await assert.rejects(() => store.pushAll([envelope], { soft: true }), /not one this runtime reads/u, `${name} soft pushAll`); + assert.throws(() => inspectCultCacheBytes(file, bytes), /not one this runtime reads/u, `${name} inspect`); + assert.deepEqual([...(await readFile(file))], [...bytes], `${name} left as it was`); + } +}); + +test("a stamp this runtime reads, followed by bytes that are not UTF-8, gets the same refusal from a read and from a whole write", async () => { + const envelope = { key: "probe", type: "probe", payload: documentPayloadFromBytes(new Uint8Array([0x80])), storedAt: "2026-10-22T00:00:00Z" }; + for (const header of ["cultcache.store.v1", "cultcache.store.v3"]) { + const stamp = [...Buffer.from(header, "utf8")]; + const forms = { fixstr: [0x91, 0xa0 | (stamp.length + 1), ...stamp, 0xff], str8: [0x91, 0xd9, stamp.length + 1, ...stamp, 0xff] }; + for (const [name, bytes] of Object.entries(forms)) { + const file = join(await mkdtemp(join(tmpdir(), "cultcache-stamp-")), "store.msgpack"); + await writeFile(file, new Uint8Array(bytes)); + const store = new SingleFileMessagePackBackingStore(file); + await assert.rejects(() => store.pullAll(), /not one this runtime reads/u, `${header} ${name} read`); + await assert.rejects(() => store.pushAll([envelope]), /not one this runtime reads/u, `${header} ${name} pushAll`); + assert.deepEqual([...(await readFile(file))], bytes, `${header} ${name} left as it was`); + } + } +}); + +test("pushAll leaves a store it reads alone when soft, and replaces one it reads otherwise", async () => { const vector = await readFile(resolve(cultLibRoot, "tests", "vectors", "document-variants-c2a", "v3-base.msgpack")); const envelope = { key: "probe", type: "probe", payload: documentPayloadFromBytes(new Uint8Array([0x80])), storedAt: "2026-10-22T00:00:00Z" }; - const newer = join(await mkdtemp(join(tmpdir(), "cultcache-pushall-soft-")), "store.msgpack"); - await writeFile(newer, vector); - await new SingleFileMessagePackBackingStore(newer).pushAll([envelope], { soft: true }); - assert.equal(Buffer.compare(await readFile(newer), vector), 0); + const existing = join(await mkdtemp(join(tmpdir(), "cultcache-pushall-soft-")), "store.msgpack"); + await writeFile(existing, vector); + await new SingleFileMessagePackBackingStore(existing).pushAll([envelope], { soft: true }); + assert.equal(Buffer.compare(await readFile(existing), vector), 0); const own = join(await mkdtemp(join(tmpdir(), "cultcache-pushall-own-")), "store.msgpack"); const store = new SingleFileMessagePackBackingStore(own); @@ -1610,6 +2225,66 @@ test("SingleFileMessagePackBackingStore reads only a missing store as empty", as assert.deepEqual(await new SingleFileMessagePackBackingStore(legacyEmpty).pullAll(), []); }); +test("reading a store that is not there writes nothing, and a read of a store that is readable does not rewrite it", async () => { + const dir = await mkdtemp(join(tmpdir(), "cultcache-read-writes-nothing-")); + const missing = join(dir, "store.cc"); + assert.deepEqual(await new SingleFileMessagePackBackingStore(missing).pullAll(), []); + assert.equal(existsSync(missing), false); + assert.deepEqual(await readdir(dir), []); + + const written = join(dir, "written.cc"); + const envelope = { key: "k", type: "t", payload: documentPayloadFromBytes(Uint8Array.of(0x90)), storedAt: "2026-09-30T00:00:00Z" }; + await new SingleFileMessagePackBackingStore(written).push(envelope); + const before = await readFile(written); + await new SingleFileMessagePackBackingStore(written).pullAll(); + assert.deepEqual(await readFile(written), before); +}); + +test("a soft pushAll writes where nothing is, leaves a readable store alone, and refuses what the reader refuses", async () => { + const dir = await mkdtemp(join(tmpdir(), "cultcache-soft-")); + const envelope = { key: "k", type: "t", payload: documentPayloadFromBytes(Uint8Array.of(0x90)), storedAt: "2026-09-30T00:00:00Z" }; + const other = { key: "other", type: "t", payload: documentPayloadFromBytes(Uint8Array.of(0x91, 0x01)), storedAt: "2026-09-30T00:00:00Z" }; + + // Absent: the soft write seeds the store. + const absent = join(dir, "absent.cc"); + await new SingleFileMessagePackBackingStore(absent).pushAll([envelope], { soft: true }); + assert.deepEqual((await new SingleFileMessagePackBackingStore(absent).pullAll()).map((entry) => entry.key), ["k"]); + + // Readable: the soft write is skipped, byte for byte. + const seeded = await readFile(absent); + await new SingleFileMessagePackBackingStore(absent).pushAll([other], { soft: true }); + assert.deepEqual(await readFile(absent), seeded); + + // Unreadable: refused, as pullAll refuses it, and the file is left exactly as it was. + const refused: Array<[string, Uint8Array]> = [ + ["zero-byte", new Uint8Array()], + ["unknown-format", encode(["cultcache.store.v9", [], []])], + ["bin-header", Uint8Array.of(0x93, 0xc4, 0x01, 0x76, 0x90, 0x90)], + ["truncated", seeded.subarray(0, seeded.length - 1)], + ["garbage", Uint8Array.of(0xc1)], + ]; + for (const [name, bytes] of refused) { + const file = join(dir, `${name}.cc`); + await writeFile(file, bytes); + await assert.rejects(() => new SingleFileMessagePackBackingStore(file).pullAll(), StoreUnreadableError, name); + await assert.rejects(() => new SingleFileMessagePackBackingStore(file).pushAll([other], { soft: true }), StoreUnreadableError, name); + assert.deepEqual(await readFile(file), Buffer.from(bytes), name); + } +}); + +test("a header that does not start at the store prefix is not echoed whole, though it ends like a version", async () => { + const dir = await mkdtemp(join(tmpdir(), "cultcache-forged-header-")); + const file = join(dir, "store.cc"); + for (const header of ["cultcache.store.SECRETcultcache.store.v9", "cultcache.store.\ncultcache.store.v9", "SECRETcultcache.store.v9"]) { + await writeFile(file, encode([header, [], []])); + await assert.rejects( + () => new SingleFileMessagePackBackingStore(file).pullAll(), + (error: Error) => error instanceof StoreUnreadableError && !error.message.includes("SECRET") && !error.message.includes(header), + header, + ); + } +}); + test("SingleFileMessagePackBackingStore reports a store it cannot reach as an I/O error", { skip: process.platform === "win32" }, async () => { const dir = await mkdtemp(join(tmpdir(), "cultcache-unreachable-")); const loop = join(dir, "loop.cc"); @@ -1658,13 +2333,39 @@ test("SingleFileMessagePackBackingStore refuses a dangling symlink at its path a await symlink(join(volume, "store.cc"), file); const store = new SingleFileMessagePackBackingStore(file); const envelope = { key: "k", type: "t", payload: documentPayloadFromBytes(Uint8Array.of(0x90)), storedAt: "2026-09-30T00:00:00Z" }; - await assert.rejects(() => store.pullAll(), (error: NodeJS.ErrnoException) => error.code === "ENOENT"); + await assert.rejects( + () => store.pullAll(), + (error: NodeJS.ErrnoException) => error.code === "ENOENT" && error.message.includes("symbolic link") && error.message.includes(file), + ); await assert.rejects(() => store.push(envelope), (error: NodeJS.ErrnoException) => error.code === "ENOENT"); await assert.rejects(() => store.pushAll([envelope], { soft: true }), (error: NodeJS.ErrnoException) => error.code === "ENOENT"); assert.ok((await lstat(file)).isSymbolicLink()); assert.deepEqual(await readdir(volume), []); }); +// The second look of the read, staged: nothing is at the path only when the read and the lstat both find nothing. A path +// that changed between them (a link in a parent swapped for a file, a store that appeared) is never an empty store. +test("readStoreBytes reports nothing at the path only when the lstat finds nothing too", async () => { + const dir = await mkdtemp(join(tmpdir(), "cultcache-second-look-")); + const file = join(dir, "store.cc"); + const failing = (code: string) => () => { + throw Object.assign(new Error(code), { code }); + }; + assert.equal(readStoreBytes(file, failing("ENOENT")), undefined); + for (const code of ["ENOTDIR", "EACCES", "ELOOP"]) { + assert.throws(() => readStoreBytes(file, failing(code)), (error: NodeJS.ErrnoException) => error.code === code, code); + } + + assert.throws( + () => readStoreBytes(file, () => ({ isSymbolicLink: () => true })), + (error: NodeJS.ErrnoException) => error.code === "ENOENT" && error.message.includes("symbolic link"), + ); + assert.throws( + () => readStoreBytes(file, () => ({ isSymbolicLink: () => false })), + (error: NodeJS.ErrnoException) => error.code === "ENOENT" && error.message.includes("appeared while it was being read"), + ); +}); + test("SingleFileMessagePackBackingStore record refusal names only a string key", async () => { const dir = await mkdtemp(join(tmpdir(), "cultcache-record-key-")); const file = join(dir, "store.cc"); @@ -1722,6 +2423,242 @@ test("a v1 store written at the base commit still reads byte for byte", async () assert.deepEqual([...documentPayloadBytes(envelopes[1]!.payload)], [0x92, 0xa4, 0x62, 0x65, 0x74, 0x61, 0x02]); }); +// The element-id marker: tests/vectors/document-variants-c2a/v3-base.msgpack is v1-base with its header replaced. +const c2aVectors = resolve(cultLibRoot, "tests", "vectors", "document-variants-c2a"); + +async function copyVector(directory: string, source: string, name: string): Promise { + const file = join(directory, name); + await writeFile(file, await readFile(source)); + return file; +} + +test("SingleFileMessagePackBackingStore pushAll applies the header the file on disk carries", async () => { + const dir = await mkdtemp(join(tmpdir(), "cultcache-v3-pushall-")); + const header = async (file: string) => (decode(await readFile(file)) as unknown[])[0]; + + // A store that never read the file still keeps a marked file marked. + const v3 = await copyVector(dir, join(c2aVectors, "v3-base.msgpack"), "v3.msgpack"); + const envelopes = await new SingleFileMessagePackBackingStore(v3).pullAll(); + await new SingleFileMessagePackBackingStore(v3).pushAll(envelopes); + assert.equal(await header(v3), "cultcache.store.v3"); + + // A store that read a marked file, whose file is then gone, writes unmarked: the disk decides, not the last read. + const reader = new SingleFileMessagePackBackingStore(v3); + await reader.pullAll(); + await rm(v3); + await reader.pushAll(envelopes); + assert.equal(await header(v3), "cultcache.store.v1"); + + const v1 = await copyVector(dir, join(variantVectors, "v1-base.msgpack"), "v1.msgpack"); + await new SingleFileMessagePackBackingStore(v1).pushAll(envelopes); + assert.equal(await header(v1), "cultcache.store.v1"); +}); + +test("SingleFileMessagePackBackingStore push after a read of a marked file that is gone writes v1", async () => { + const dir = await mkdtemp(join(tmpdir(), "cultcache-v3-gone-")); + const v3 = await copyVector(dir, join(c2aVectors, "v3-base.msgpack"), "v3.msgpack"); + const store = new SingleFileMessagePackBackingStore(v3); + const envelopes = await store.pullAll(); + await rm(v3); + await store.push(envelopes[0]!); + assert.equal((decode(await readFile(v3)) as unknown[])[0], "cultcache.store.v1"); +}); + +test("SingleFileMessagePackBackingStore reads a v3 element-id store and a rewrite keeps the marker", async () => { + const dir = await mkdtemp(join(tmpdir(), "cultcache-v3-")); + const v3 = await copyVector(dir, join(c2aVectors, "v3-base.msgpack"), "v3.msgpack"); + const store = new SingleFileMessagePackBackingStore(v3); + const envelopes = await store.pullAll(); + assert.deepEqual(envelopes.map((entry) => entry.key), ["alpha", "beta"]); + assert.deepEqual([...documentPayloadBytes(envelopes[0]!.payload)], [0x92, 0xa5, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x01]); + await store.push({ ...envelopes[0]!, storedAt: "2026-09-30T00:00:00.0000000Z" }); + assert.equal((decode(await readFile(v3)) as unknown[])[0], "cultcache.store.v3"); + + // A v1 store stays v1: the marker is carried, never invented. + const v1 = await copyVector(dir, join(variantVectors, "v1-base.msgpack"), "v1.msgpack"); + const plain = new SingleFileMessagePackBackingStore(v1); + const plainEnvelopes = await plain.pullAll(); + await plain.push({ ...plainEnvelopes[0]!, storedAt: "2026-09-30T00:00:00.0000000Z" }); + assert.equal((decode(await readFile(v1)) as unknown[])[0], "cultcache.store.v1"); + + const inspection = inspectCultCacheBytes("v3.msgpack", await readFile(join(c2aVectors, "v3-base.msgpack"))); + assert.equal(inspection.format, "cultcache.store.v3"); +}); + +test("SingleFileMessagePackBackingStore pushAll refuses a store whose header it cannot read and leaves the file untouched", async () => { + const dir = await mkdtemp(join(tmpdir(), "cultcache-pushall-refused-")); + const envelopes = await new SingleFileMessagePackBackingStore( + await copyVector(dir, join(c2aVectors, "v3-base.msgpack"), "seed.msgpack"), + ).pullAll(); + for (const vector of [ + join(variantVectors, "unknown-header.msgpack"), + join(variantVectors, "variant-v2.msgpack"), + resolve(cultLibRoot, "tests", "vectors", "document-variants-c1", "variant-store.msgpack"), + ]) { + const file = await copyVector(dir, vector, "store.msgpack"); + const before = await readFile(file); + await assert.rejects( + () => new SingleFileMessagePackBackingStore(file).pushAll(envelopes), + (error) => error instanceof StoreUnreadableError && /is not readable/u.test(error.message), + vector, + ); + assert.deepEqual(await readFile(file), before, `${vector} was rewritten`); + } +}); + +// A file is replaced by a rewrite exactly when this runtime's own reader opens it: one verdict per file, asked by pullAll, +// pushAll, soft pushAll, push and delete alike; a soft flush leaves the files that read as they are. The bytes and every runtime's verdict: tests/vectors/document-variants-c2a/readability. +test("SingleFileMessagePackBackingStore replaces a file exactly when it reads", async () => { + const root = join(c2aVectors, "readability"); + const rows = (await readFile(join(root, "manifest.txt"), "utf8")) + .split(/\r?\n/u) + .filter((line) => line.length > 0 && !line.startsWith("#")) + .map((line) => line.split(/\s+/u)); + // Every vector in the folder has a manifest row. + const listed = rows.map(([vector]) => vector!).filter((vector) => !vector.startsWith("..")).sort(); + assert.deepEqual((await readdir(root)).filter((name) => name.endsWith(".bin")).sort(), listed); + const dir = await mkdtemp(join(tmpdir(), "cultcache-readability-")); + const seed = await copyVector(dir, join(c2aVectors, "v3-base.msgpack"), "seed.msgpack"); + const envelopes = await new SingleFileMessagePackBackingStore(seed).pullAll(); + for (const [vector, , , typescript] of rows) { + const reads = typescript === "reads"; + const bytes = await readFile(join(root, vector)); + for (const operation of ["pullAll", "softPushAll", "pushAll", "push", "delete"] as const) { + const file = join(dir, `${vector.replace(/[^a-z0-9.-]/giu, "_")}.${operation}.msgpack`); + await writeFile(file, bytes); + const store = new SingleFileMessagePackBackingStore(file); + const run = () => + operation === "pullAll" + ? store.pullAll() + : operation === "softPushAll" + ? store.pushAll(envelopes, { soft: true }) + : operation === "pushAll" + ? store.pushAll(envelopes) + : operation === "push" + ? store.push(envelopes[0]!) + : store.delete(envelopes[0]!); + if (reads) { + await run(); + if (operation === "pullAll") { + continue; + } + + // A soft flush leaves a store it reads byte for byte: nothing is written where something is. + if (operation === "softPushAll") { + assert.ok(bytes.equals(await readFile(file)), `${vector} ${operation} rewrote a store it reads`); + continue; + } + + // The replaced file is a store: it carries the header its old content decides (a v3 store keeps its marker, all else + // is v1), and it holds what the operation wrote. + assert.equal((decode(await readFile(file)) as unknown[])[0], vector.includes("v3") ? "cultcache.store.v3" : "cultcache.store.v1", `${vector} ${operation}`); + const keys = (await new SingleFileMessagePackBackingStore(file).pullAll()).map((entry) => entry.key); + if (operation === "pushAll") { + assert.deepEqual(keys, envelopes.map((entry) => entry.key).sort(), `${vector} ${operation}`); + } else if (operation === "push") { + assert.ok(keys.includes(envelopes[0]!.key), `${vector} ${operation}`); + } else { + assert.ok(!keys.includes(envelopes[0]!.key), `${vector} ${operation}`); + } + } else { + await assert.rejects( + run, + (error) => error instanceof StoreUnreadableError && error.cause !== undefined, + `${vector} ${operation}`, + ); + assert.ok(bytes.equals(await readFile(file)), `${vector} ${operation} rewrote a file it cannot read`); + } + } + } +}); + +// Owner beats lister: a definition that owns an id and one that lists it as compatible register together in either order. A record +// under the id is the owner's, and a lister's only when nothing owns it; two listers and no owner name no single definition. +const ownsId = defineDocumentType({ type: "tests.owns", schema: z.object({ name: z.string() }), schemaId: "id.owned", schemaName: "tests.owns" }); +const listsId = defineDocumentType({ + type: "tests.lists", + schema: z.object({ name: z.string() }), + schemaId: "id.lister", + schemaName: "tests.lists", + compatibleSchemaIds: ["id.owned"], +}); +const alsoListsId = defineDocumentType({ + type: "tests.also-lists", + schema: z.object({ name: z.string() }), + schemaId: "id.also-lister", + schemaName: "tests.also-lists", + compatibleSchemaIds: ["id.owned"], +}); + +async function storeUnder(records: [string, string][]): Promise { + const seed = CultCache.builder().withDocumentType(ownsId).build(); + await seed.put(ownsId, "seed", { name: "n" }); + const payload = seed.getRequiredEnvelope(ownsId, "seed").payload; + const file = join(await mkdtemp(join(tmpdir(), "cultcache-owner-lister-")), "store.msgpack"); + const ids = [...new Set(records.map(([, schemaId]) => schemaId))]; + await writeFile(file, encode(["cultcache.store.v1", + ids.map((schemaId) => [schemaId, schemaId, `${schemaId}.v1`, schemaId, "", [schemaId], []]), + records.map(([key, schemaId]) => [key, schemaId, "2026-09-30T00:00:00.0000000Z", payload])])); + return file; +} + +async function openWith(file: string, ...definitions: AnyCultCacheDocumentDefinition[]): Promise { + let builder = CultCache.builder(); + for (const definition of definitions) builder = builder.withDocumentType(definition); + const cache = builder.withGenericStore(new SingleFileMessagePackBackingStore(file)).build(); + await cache.pullAllBackingStores(); + return cache; +} + +test("a definition listing an id registers beside the one owning it in either order, and the owner wins", async () => { + const file = await storeUnder([["o", "id.owned"], ["l", "id.lister"]]); + for (const order of [[ownsId, listsId], [listsId, ownsId]]) { + const cache = await openWith(file, ...order); + assert.deepEqual(cache.getRequired(ownsId, "o"), { name: "n" }); + assert.equal(cache.get(listsId, "o"), undefined); + assert.deepEqual(cache.getRequired(listsId, "l"), { name: "n" }); + } + const listerOnly = await openWith(await storeUnder([["o", "id.owned"]]), listsId); + assert.deepEqual(listerOnly.getRequired(listsId, "o"), { name: "n" }); +}); + +test("two definitions listing one id register, and a record under it needs its owner", async () => { + const file = await storeUnder([["k", "id.owned"]]); + for (const listers of [[listsId, alsoListsId], [alsoListsId, listsId]]) { + await assert.rejects(openWith(file, ...listers), (error: unknown) => { + assert.ok(error instanceof SchemaConflictError); + assert.equal(error.schemaId, "id.owned"); + assert.equal(error.recordKey, "k"); + assert.deepEqual(error.schemaNames, ["tests.also-lists", "tests.lists"]); + return true; + }); + for (const order of [[...listers, ownsId], [ownsId, ...listers]]) { + assert.deepEqual((await openWith(file, ...order)).getRequired(ownsId, "k"), { name: "n" }); + } + } +}); + +test("every registration refusal is a SchemaConflictError naming the id and both schema names", () => { + const sameId = defineDocumentType({ type: "tests.owns-too", schema: z.object({ name: z.string() }), schemaId: "id.owned", schemaName: "tests.owns-too" }); + const sameName = defineDocumentType({ type: "tests.named-too", schema: z.object({ name: z.string() }), schemaId: "id.other", schemaName: "tests.owns" }); + const sameType = defineDocumentType({ type: "tests.owns", schema: z.object({ name: z.string() }), schemaId: "id.third", schemaName: "tests.third" }); + for (const [claimant, schemaId, names] of [ + [sameId, "id.owned", ["tests.owns", "tests.owns-too"]], + [sameName, "id.other", ["tests.owns", "tests.owns"]], + [sameType, "id.third", ["tests.owns", "tests.third"]], + ] as const) { + for (const order of [[ownsId, claimant], [claimant, ownsId]]) { + assert.throws(() => CultCache.builder().withDocumentType(order[0]).withDocumentType(order[1]).build(), (error: unknown) => { + assert.ok(error instanceof SchemaConflictError, String(error)); + const first = order[0] === ownsId; + assert.equal(error.schemaId, first ? schemaId : ownsId.schemaId); + assert.deepEqual(error.schemaNames, first ? names : [...names].reverse()); + return true; + }); + } + } +}); + // --------------------------------------------------------------------------------------------- // Cross-runtime store lock: each runtime's writer excludes the others on `.lock`. diff --git a/packages/cultcache-ts/test/payload-depth.test.ts b/packages/cultcache-ts/test/payload-depth.test.ts index 02e55ab3..382f0672 100644 --- a/packages/cultcache-ts/test/payload-depth.test.ts +++ b/packages/cultcache-ts/test/payload-depth.test.ts @@ -20,7 +20,6 @@ import { } from "../src/document"; import * as codecs from "../src/codec"; import { checkPayloadDepth } from "../src/payload-depth"; -import { SingleFileMessagePackBackingStore } from "../src/single-file-messagepack-backing-store"; const L = MAX_PAYLOAD_DEPTH; const CANARY = "CANARY-payload-depth-91d3"; @@ -175,11 +174,11 @@ test("the limit and its error are exported from the root, and the walk itself is assert.equal("checkPayloadDepth" in root, false); }); -// A v1 store whose one record carries `payload` as bin and no catalog entry, so the record's schema is inferred from the -// payload's first element. +// A v1 store whose one record carries `payload` as bin, under a catalog entry that names its schema. async function storeWith(payload: Uint8Array): Promise { const file = join(await mkdtemp(join(tmpdir(), "cultcache-payload-depth-")), "deep.cc"); - await writeFile(file, encode(["cultcache.store.v1", [], [["record-1", "sha256:stale", "2026-06-25T12:00:00Z", payload]]])); + const entry = ["sha256:stale", "tests.schema_stamped_entry", "tests.schema_stamped_entry.v1", "stale", "", [], []]; + await writeFile(file, encode(["cultcache.store.v1", [entry], [["record-1", "sha256:stale", "2026-06-25T12:00:00Z", payload]]])); return file; } const stamped = (containersInsideTheStamp: number) => @@ -189,25 +188,16 @@ test("the inspector previews a payload at the limit and reports one past it, or const atLimit = await storeWith(stamped(L - 1)); const previewed = inspectCultCacheBytes(atLimit, await readFile(atLimit)).records[0]!; assert.equal(previewed.payloadDecodeError, undefined); - assert.equal(previewed.schemaName, "tests.schema_stamped_entry", "a payload the walk accepts yields its stamp"); + assert.equal(previewed.schemaName, "tests.schema_stamped_entry"); for (const inside of [L, 100_000]) { const file = await storeWith(stamped(inside)); const record = inspectCultCacheBytes(file, await readFile(file)).records[0]!; assert.equal(record.payloadDecodeError, "document payload nests 257 MessagePack containers deep; the limit is 256."); - assert.equal(record.schemaName, "", "the stamp of a payload the walk refuses is not recovered"); + assert.equal(record.schemaName, "tests.schema_stamped_entry"); } }); -test("the single-file store infers a legacy record's schema from a payload at the limit and from none past it", async () => { - const atLimit = await storeWith(stamped(L - 1)); - const [read] = await new SingleFileMessagePackBackingStore(atLimit).pullAll(); - assert.equal(read.catalogEntry?.schemaVersion, "tests.schema_stamped_entry.v1"); - - const past = await storeWith(stamped(L)); - await assert.rejects(() => new SingleFileMessagePackBackingStore(past).pullAll(), /missing schema id/u); -}); - // A header whose length or count needs all four bytes: the low sixteen bits of the length are small, the whole is not. const be32 = (n: number) => bytes((n >>> 24) & 0xff, (n >>> 16) & 0xff, (n >>> 8) & 0xff, n & 0xff); diff --git a/packages/cultcache-ts/test/union-parity.test.ts b/packages/cultcache-ts/test/union-parity.test.ts index f3e79b37..46dcb42f 100644 --- a/packages/cultcache-ts/test/union-parity.test.ts +++ b/packages/cultcache-ts/test/union-parity.test.ts @@ -48,16 +48,23 @@ type Node = | { kind: "Text"; id: string; content: string } | { kind: "Use"; id: string; composition: Composition; slots: (Node | null)[] } | { kind: "Image"; id: string; uri: string; alt: string } - | { kind: "Wrap"; inner: Node | null }; + | { kind: "Wrap"; inner: Node | null; id: string }; interface Composition { name: string; body: Node | null } type Outcome = { kind: "Pending"; message: string } | { kind: "Completed"; sourceVersion: number } | { kind: "Denied" } | { kind: "Floor" } | { kind: "Ceiling" }; +// The golden's Outcome: the arms of a list carry an element id, so Denied, Floor and Ceiling are one-slot records. +type GoldenOutcome = + | { kind: "Pending"; message: string; id: string } + | { kind: "Completed"; sourceVersion: number; id: string } + | { kind: "Denied"; id: string } + | { kind: "Floor"; id: string } + | { kind: "Ceiling"; id: string }; type Item = { kind: "Gem"; name: string } | { kind: "Tool"; name: string; uses: number } | { kind: "Part"; name: string; count: number }; type CraftedItem = { kind: "Tool"; name: string; uses: number } | { kind: "Part"; name: string; count: number }; interface Surface { id: string; version: number; root: Node | null; - outcomes: (Outcome | null)[]; + outcomes: (GoldenOutcome | null)[]; part: Item | null; crafted: CraftedItem | null; missing: Node | null; @@ -71,7 +78,7 @@ const node: Codec = union("Node", { 0: ["id", string], 1: ["composition", composition], 2: ["slots", list(lazy(() => node))], })], 5: ["Image", slots<{ id: string; uri: string; alt: string }>({ 0: ["id", string], 1: ["uri", string], 3: ["alt", string] })], - 6: ["Wrap", slots<{ inner: Node | null }>({ 0: ["inner", lazy(() => node)] })], + 6: ["Wrap", slots<{ inner: Node | null; id: string }>({ 0: ["inner", lazy(() => node)], 1: ["id", string] })], }, `${CS}Node`); const outcome = union("Outcome", { 0: ["Pending", slots<{ message: string }>({ 0: ["message", string] })], @@ -80,6 +87,14 @@ const outcome = union("Outcome", { [-2147483648]: ["Floor", unit], 2147483647: ["Ceiling", unit], }, `${CS}Outcome`); +const idOnly = slots<{ id: string }>({ 0: ["id", string] }); +const goldenOutcome = union("Outcome", { + 0: ["Pending", slots<{ message: string; id: string }>({ 0: ["message", string], 1: ["id", string] })], + 1: ["Completed", slots<{ sourceVersion: number; id: string }>({ 0: ["sourceVersion", int32], 1: ["id", string] })], + 2: ["Denied", idOnly], + [-2147483648]: ["Floor", idOnly], + 2147483647: ["Ceiling", idOnly], +}, `${CS}Outcome`); const toolArm = slots<{ name: string; uses: number }>({ 0: ["name", string], 1: ["uses", int32] }); const partArm = slots<{ name: string; count: number }>({ 0: ["name", string], 1: ["count", int32] }); const item = union("Item", { @@ -92,7 +107,7 @@ const surfaceCodec = slots({ 0: ["id", string], 1: ["version", int32], 2: ["root", node], - 3: ["outcomes", list(outcome)], + 3: ["outcomes", list(goldenOutcome)], 4: ["part", item], 5: ["crafted", craftedItem], 6: ["missing", node, null], @@ -111,7 +126,7 @@ const nodeSchema: z.ZodType = z.lazy(() => slots: z.array(nodeSchema), }), z.object({ kind: z.literal("Image"), id: z.string(), uri: z.string(), alt: z.string() }), - z.object({ kind: z.literal("Wrap"), inner: nodeSchema }), + z.object({ kind: z.literal("Wrap"), inner: nodeSchema, id: z.string() }), ]), ])); const surfaceSchema = z.object({ @@ -121,11 +136,11 @@ const surfaceSchema = z.object({ outcomes: z.array(z.union([ z.null(), z.discriminatedUnion("kind", [ - z.object({ kind: z.literal("Pending"), message: z.string() }), - z.object({ kind: z.literal("Completed"), sourceVersion: z.number() }), - z.object({ kind: z.literal("Denied") }), - z.object({ kind: z.literal("Floor") }), - z.object({ kind: z.literal("Ceiling") }), + z.object({ kind: z.literal("Pending"), message: z.string(), id: z.string() }), + z.object({ kind: z.literal("Completed"), sourceVersion: z.number(), id: z.string() }), + z.object({ kind: z.literal("Denied"), id: z.string() }), + z.object({ kind: z.literal("Floor"), id: z.string() }), + z.object({ kind: z.literal("Ceiling"), id: z.string() }), ]), ])), part: z.union([z.null(), z.discriminatedUnion("kind", [ @@ -157,29 +172,33 @@ const golden: Surface = { version: 3, root: { kind: "Box", - id: "root", + id: "0000000000c0", children: [ - { kind: "Text", id: "t1", content: "hello" }, + { kind: "Text", id: "0000000000c1", content: "hello" }, { kind: "Use", - id: "u1", + id: "0000000000c2", composition: { name: "comp", body: { kind: "Box", - id: "inner", + id: "0000000000d0", children: [ - { kind: "Text", id: "deep", content: "x" }, - { kind: "Image", id: "img", uri: "uri://a", alt: "alt" }, + { kind: "Text", id: "0000000000d1", content: "x" }, + { kind: "Image", id: "0000000000d2", uri: "uri://a", alt: "alt" }, ], }, }, - slots: [{ kind: "Text", id: "s1", content: "slot" }, null, { kind: "Box", id: "b2", children: [] }], + slots: [{ kind: "Text", id: "0000000000e1", content: "slot" }, null, { kind: "Box", id: "0000000000e2", children: [] }], }, - { kind: "Image", id: "i2", uri: "uri://b", alt: "" }, + { kind: "Image", id: "0000000000c3", uri: "uri://b", alt: "" }, ], }, - outcomes: [{ kind: "Pending", message: "wait" }, { kind: "Completed", sourceVersion: 7 }, { kind: "Denied" }], + outcomes: [ + { kind: "Pending", message: "wait", id: "0000000000a1" }, + { kind: "Completed", sourceVersion: 7, id: "0000000000a2" }, + { kind: "Denied", id: "0000000000a3" }, + ], part: { kind: "Tool", name: "hammer", uses: 12 }, crafted: { kind: "Part", name: "gear", count: 4 }, missing: null, @@ -200,8 +219,13 @@ test("the union surface vector decodes to the constructed value and re-encodes b assert.equal(hex(writeWire(surfaceCodec.encode(golden))), hex(committed)); }); -test("a unit arm is written [key, []] and its payload must be an array", () => { - assert.deepEqual(outcome.encode({ kind: "Denied" }), [2, []]); +test("a unit arm is written as the C# reference writes it, and its payload must be an array", async () => { + // The bytes are tests/vectors/union-parity/unit-arm-*.msgpack, written by the reference, not spelled here. + for (const [file, kind] of [["denied", "Denied"], ["floor", "Floor"], ["ceiling", "Ceiling"]] as const) { + const reference = await read(`unit-arm-${file}.msgpack`); + assert.deepEqual(outcome.decode(readWire(reference)), { kind }, file); + assert.equal(hex(writeWire(outcome.encode({ kind }))), hex(reference), file); + } assert.throws(() => outcome.decode([2, null]), /Union Outcome/u); }); @@ -814,14 +838,14 @@ test("a bytes slot reads a MessagePack str as its bytes and writes bin, and refu const wrapChain = (wraps: number, leaf: Node): Node => { let node: Node = leaf; - for (let index = 0; index < wraps; index += 1) node = { kind: "Wrap", inner: node }; + for (let index = 0; index < wraps; index += 1) node = { kind: "Wrap", inner: node, id: "0000000000b1" }; return node; }; // A Wrap adds 2 containers, a Box with no children 3, a Text leaf 2, and the outermost array is 1 (the C# test's arithmetic). const AT_LIMIT_WRAPS = (MAX_PAYLOAD_DEPTH - 1 - 3) / 2; const PAST_LIMIT_WRAPS = (MAX_PAYLOAD_DEPTH + 1 - 1 - 2) / 2; -const atLimitValue: Surface = { ...golden, root: wrapChain(AT_LIMIT_WRAPS, { kind: "Box", id: "leaf", children: [] }) }; -const pastLimitValue: Surface = { ...golden, root: wrapChain(PAST_LIMIT_WRAPS, { kind: "Text", id: "leaf", content: CANARY }) }; +const atLimitValue: Surface = { ...golden, root: wrapChain(AT_LIMIT_WRAPS, { kind: "Box", id: "0000000000f0", children: [] }) }; +const pastLimitValue: Surface = { ...golden, root: wrapChain(PAST_LIMIT_WRAPS, { kind: "Text", id: "0000000000f0", content: CANARY }) }; async function boundsEntries() { const raw = decode(await read("bounds-outcomes.msgpack")) as unknown[][]; diff --git a/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/CultMesh.kt b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/CultMesh.kt index 368b5338..10cc2a58 100644 --- a/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/CultMesh.kt +++ b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/CultMesh.kt @@ -56,6 +56,17 @@ data class CultCacheEnvelope( val payload: ByteArray, ) +/** + * A schema this cache cannot give to one more type: another type already holds the schema id, or the type already holds + * another schema id. Nothing is registered. It names the refused codec's schema id and both document types. + */ +class CultSchemaConflictException( + val schemaId: String, + val schemaNames: List, + val recordKey: String, + message: String, +) : IOException(message) + class CultCache { companion object { const val GLOBAL_KEY = "__global__" @@ -65,7 +76,19 @@ class CultCache { private val codecsBySchema = linkedMapOf>() private val values = linkedMapOf>() + // One type owns a schema id. Every path that names a codec (register, put, get, getAll, delete) comes through here. fun register(codec: CultDocumentCodec) { + val holder = codecs[codec.documentType]?.takeIf { it.schemaVersion != codec.schemaVersion } + ?: codecsBySchema[codec.schemaVersion]?.takeIf { it.documentType != codec.documentType } + if (holder != null) { + throw CultSchemaConflictException( + codec.schemaVersion, + listOf(holder.documentType, codec.documentType), + "", + "Kotlin CultCache type ${holder.documentType} holds schema ${holder.schemaVersion}; " + + "type ${codec.documentType} cannot also be registered for schema id ${codec.schemaVersion}", + ) + } codecs[codec.documentType] = codec codecsBySchema[codec.schemaVersion] = codec } @@ -3555,6 +3578,42 @@ private fun cultCacheRawSnapshotsRoundTripThroughCultNetMessages() { check(syncedAlias == KotlinUiNote("kotlin.alias_note.v1", "canonical-to-ui")) check(target.require(uiNote, "note:alias") == syncedAlias) + // One type per schema in one cache: a second type for a held schema id, or a held type for another schema id, is + // refused typed, by register and by the implicit registration in put, get, getAll and delete alike. + fun refusal(action: () -> Unit): CultSchemaConflictException? = + try { action(); null } catch (conflict: CultSchemaConflictException) { conflict } + val canonicalOtherSchema = cultDocument(KotlinAliasNoteCodec( + documentType = "kotlin.alias_note", + schemaVersion = "kotlin.alias_note.v2", + create = ::KotlinCanonicalNote, + readSchemaVersion = KotlinCanonicalNote::schemaVersion, + readBody = KotlinCanonicalNote::body, + )) + for ((first, second) in listOf(canonicalNote to uiNote, uiNote to canonicalNote, canonicalNote to canonicalOtherSchema)) { + @Suppress("UNCHECKED_CAST") + val claimant = second as CultDocumentDefinition + val claimantValue = claimant.codec.decode(canonicalNote.codec.encode(KotlinCanonicalNote("kotlin.alias_note.v1", "x"))) + val attempts: List<(CultCache) -> Unit> = listOf( + { cache -> cache.register(claimant) }, + { cache -> cache.put(claimant, "note:second", claimantValue) }, + { cache -> cache.get(claimant, "note:second") }, + { cache -> cache.getAll(claimant) }, + { cache -> cache.delete(claimant.codec, "note:second") }, + ) + for (attempt in attempts) { + val cache = CultCache() + @Suppress("UNCHECKED_CAST") + cache.register(first as CultDocumentDefinition) + val conflict = refusal { attempt(cache) } + check(conflict != null) { "${second.documentType}/${second.schemaVersion} beside ${first.documentType}/${first.schemaVersion} was not refused" } + check(conflict.schemaId == second.schemaVersion && conflict.recordKey == "") + check(conflict.schemaNames == listOf(first.documentType, second.documentType)) + check(conflict.message!!.contains(second.schemaVersion)) + check(cache.codecForSchema(first.schemaVersion).documentType == first.documentType) + } + } + CultCache().apply { register(canonicalNote); register(canonicalNote); put(canonicalNote, "note:again", KotlinCanonicalNote("kotlin.alias_note.v1", "again")) } + val reactiveUiNote = cultDocument(KotlinReactiveNoteCodec( documentType = "kotlin.reactive_note.ui", schemaVersion = "kotlin.reactive_note.v1", @@ -3580,8 +3639,8 @@ private fun cultCacheRawSnapshotsRoundTripThroughCultNetMessages() { } val reactiveCanonicalNote = cultDocument(KotlinReactiveNoteCodec( - documentType = "kotlin.reactive_note", - schemaVersion = "kotlin.reactive_note.v1", + documentType = "kotlin.reconcile_note", + schemaVersion = "kotlin.reconcile_note.v1", )) target.cache.register(reactiveCanonicalNote) target.remember(reactiveCanonicalNote, "note:reconcile", KotlinReactiveNote("initial", 1)) diff --git a/packages/cultmesh-py/src/cultmesh_py/node.py b/packages/cultmesh-py/src/cultmesh_py/node.py index bf092060..d251a030 100644 --- a/packages/cultmesh-py/src/cultmesh_py/node.py +++ b/packages/cultmesh-py/src/cultmesh_py/node.py @@ -26,7 +26,7 @@ from cultnet_py import ( apply_shard_log_response as apply_cultnet_shard_log_response, document_delete, document_put_raw, - resolve_document_and_schema_id_for_raw_record, + resolve_document_for_raw_record, schema_document_map, ) @@ -689,17 +689,14 @@ class CultMeshDatabase: schema_id = str(document_record.get("schemaId") or "") documents_by_schema_id = schema_document_map(self.documents) try: - document, resolved_schema_id = resolve_document_and_schema_id_for_raw_record( - documents_by_schema_id, - schema_id, - document_record, - ) + document = resolve_document_for_raw_record(documents_by_schema_id, schema_id, document_record) except KeyError: return None + # One id: the local schema id, under which the cache stores the record. return document, CultCacheEnvelope( key=str(document_record["recordKey"]), type=document.type, - schema_id=resolved_schema_id, + schema_id=document.catalog_entry().schema_id, payload=bytes(document_record["payload"]), stored_at=str(document_record.get("storedAt") or ""), catalog_entry=document.catalog_entry(), @@ -1168,12 +1165,8 @@ class CultMeshDatabase: documents_by_schema_id = schema_document_map(self.documents) if raw_record is not None: try: - _, resolved_schema_id = resolve_document_and_schema_id_for_raw_record( - documents_by_schema_id, - schema_id, - raw_record, - ) - return resolved_schema_id, record_key + document = resolve_document_for_raw_record(documents_by_schema_id, schema_id, raw_record) + return document.catalog_entry().schema_id, record_key except KeyError: pass document = documents_by_schema_id.get(schema_id) @@ -1333,15 +1326,11 @@ class CultMeshDatabase: continue schema_id = str(record.get("schemaId")) try: - document, resolved_schema_id = resolve_document_and_schema_id_for_raw_record( - documents_by_schema_id, - schema_id, - record, - ) + document = resolve_document_for_raw_record(documents_by_schema_id, schema_id, record) except KeyError: continue record_key = str(record.get("recordKey")) - previous[(resolved_schema_id, record_key)] = self.cache.get(document, record_key) + previous[(document.catalog_entry().schema_id, record_key)] = self.cache.get(document, record_key) return previous @staticmethod diff --git a/packages/cultmesh-py/tests/test_cultmesh.py b/packages/cultmesh-py/tests/test_cultmesh.py index cdf2c965..67f5fe36 100644 --- a/packages/cultmesh-py/tests/test_cultmesh.py +++ b/packages/cultmesh-py/tests/test_cultmesh.py @@ -380,6 +380,46 @@ class CultMeshTests(unittest.TestCase): self.assertEqual(observed[0].value["value"], "new") self.assertEqual(node.cache.get_required_envelope(document, "policy:1").schema_id, local_schema_id) + # A record arriving under a compatible id or the schema name is the same record: one id, the local one, so an existing + # record is updated with its previous value. + def test_cultmesh_node_reports_an_update_when_a_record_arrives_under_a_compatible_id_or_the_schema_name(self) -> None: + document = define_database_entry_type( + "mesh.compatible-policy", + [("schema_version", 0), ("name", 1), ("value", 2)], + schema_id="mesh.compatible-policy.current", + schema_name="mesh.compatible_policy", + schema_version="mesh.compatible_policy.v1", + compatible_schema_ids=["mesh.compatible-policy.current", "mesh.compatible-policy.older"], + ) + for wire_schema_id in ("mesh.compatible-policy.older", "mesh.compatible_policy"): + node = create_node(runtime_id="python-node") + node.register_document(document) + node.put(document, "policy:1", {"schema_version": "mesh.compatible_policy.v1", "name": "policy", "value": "old"}) + observed: list[CultMeshDatabaseChange] = [] + unsubscribe = node.database.watch_record(document, "policy:1", observed.append) + try: + applied = node.database.apply_snapshot_response({ + "schemaVersion": "cultnet.snapshot_response_raw.v0", + "messageId": "compatible-snapshot", + "documents": [{ + "schemaId": wire_schema_id, + "recordKey": "policy:1", + "storedAt": "2026-06-25T00:00:00Z", + "payloadEncoding": "messagepack", + "payload": document.encode_payload({ + "schema_version": "mesh.compatible_policy.v1", "name": "policy", "value": "new", + }), + }], + }) + finally: + unsubscribe() + + local_schema_id = document.catalog_entry().schema_id + self.assertEqual(applied[0].schema_id, local_schema_id, wire_schema_id) + self.assertEqual([change.change_kind for change in observed], ["updated"], wire_schema_id) + self.assertEqual(observed[0].previous_value["value"], "old", wire_schema_id) + self.assertEqual(observed[0].value["value"], "new", wire_schema_id) + def test_cultmesh_node_syncs_snapshot_and_shard_log_through_cultnet_client(self) -> None: import msgpack # type: ignore from cultnet_py import read_frame, write_frame diff --git a/packages/cultmesh-rs/src/lib.rs b/packages/cultmesh-rs/src/lib.rs index 6965e2d5..e50719cd 100644 --- a/packages/cultmesh-rs/src/lib.rs +++ b/packages/cultmesh-rs/src/lib.rs @@ -62,7 +62,7 @@ macro_rules! cultmesh_documents { $schema_version.to_string(), $schema_version.to_string(), ), - ); + )?; )* Ok(()) } diff --git a/packages/cultmesh-ts/CHANGELOG.md b/packages/cultmesh-ts/CHANGELOG.md index 4a43ec61..8b4e0ab5 100644 --- a/packages/cultmesh-ts/CHANGELOG.md +++ b/packages/cultmesh-ts/CHANGELOG.md @@ -3,6 +3,23 @@ All notable changes to `cultmesh-ts` are recorded here, following [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). +## Unreleased + +### Breaking + +- `asSchemaAlias` is removed from `CultMeshDocumentHandle`, `CultMeshCollectionHandle` and their bound forms. Read the handle by the schema you mean. +- `CultMeshDocumentCatalog` (`document`, `tryDocument`, `latest`, `replace`, `submitPrediction`, `authoritativeWriter`, `predictionWriter`, `watch`) and `CultMeshCollectionCatalog` (`collection`, `tryCollection`, `latest`, `watchChanges`) take `` and yield `CultMeshCatalogValue`: an explicit type argument wins, a CultCache document definition names its own value type, and a bare schema descriptor yields `unknown`. A value type is no longer inferred from a value or an expected result. The `parse` option is removed from these calls, and a lookup by type answers with that type's handle or none, not an alias of a compatible schema. +- Adding a document or collection to a catalog under a schema id or schema name another type already holds is refused with `SchemaConflictError`. + +### Added + +- `CultMeshCatalogValue`, exported. +- `documentById` and `tryDocumentById` on `CultMeshDocumentCatalog`, which address one document by its document id when a catalog holds several documents of one type. + +### Dependencies + +- Needs the cultcache-ts release that ships with this change (its manifest range moves with that release): it imports `schemaIdentityOf` and `SchemaConflictError`, which 0.14.x does not export. + ## [0.2.0] ### Added diff --git a/packages/cultmesh-ts/README.md b/packages/cultmesh-ts/README.md index de7b2794..2f9eda51 100644 --- a/packages/cultmesh-ts/README.md +++ b/packages/cultmesh-ts/README.md @@ -87,16 +87,16 @@ await input.submitPrediction("pilot-a", predictedInput); const stop = input.watch("pilot-a", latest => reconcileInput(latest)); ``` -Durable nodes resolve same-schema aliases at the boundary. Register the -canonical document that owns storage and replication, then ask the node for the -typed view the current runtime wants: +Durable nodes hand out typed handles for the documents they register. One type +owns each schema: a node refuses a definition it did not register, and refuses +to register a second type for a schema it already holds. ```ts const station = await CultMesh.startNode(statePath, { - documents: [stationStockDocument], + documents: [stationStockDocument, daemonHealthDocument], }); -const stock = station.document(stationStockUiDocument, "station:starbridge:stock"); +const stock = station.document(stationStockDocument, "station:starbridge:stock"); const current = await stock.latest(); await stock.authoritativeWriter().write(updatedStockFromUi); @@ -108,6 +108,20 @@ reactive.update(draft => { }); ``` +`node.get`, `node.getRequired` and the value `node.put` resolves to return the +object the node's cache holds, not a copy: two reads of one record return the +same object. That object belongs to the cache and must not be mutated. Mutating +it changes what later reads see without writing anything to the store. To +change a document, build a new value (or use a writer) and put it: + +```ts +const held = station.getRequired(stationStockDocument, "station:starbridge:stock"); +await station.put(stationStockDocument, "station:starbridge:stock", { + ...held, + availableMissiles: held.availableMissiles - 4, +}); +``` + State pointers are the same kind of managed surface for UI and tools. They can advertise source documents, inherit a Verse route when bound, and resolve through that Verse without caller-side context plumbing: @@ -386,14 +400,14 @@ const latestHealth = await health.latest(); const syncedHealth = await station.syncDocumentFromPeerSnapshot( () => client, - daemonHealthUiDocument, + daemonHealthDocument, "daemon:aetheria.health.v1", { timeoutMs: 5_000, }, ); const localHealth = await station - .document(daemonHealthUiDocument, "daemon:aetheria.health.v1") + .document(daemonHealthDocument, "daemon:aetheria.health.v1") .latest(); ``` @@ -409,11 +423,11 @@ before using the peer-card endpoint as a dial target. snapshots. It hides the snapshot request message id, response listener, raw payload binary normalization, and MessagePack decode behind the same `CultMeshDocumentHandle` shape as local documents. The helper prefers an exact -schema id match and falls back to the requested record key, so runtimes that -alias the same logical document through different generated schema names can -still read the publication with one CultMesh call. The handle takes a CultCache -document definition, and its `latest()` result is read and parsed through that -definition: a payload is a `DocumentPayload`, which only +schema id match, else a record at the requested key under an id the +definition declares compatible; a record under any other schema id is another +type's and is never read in its place, whatever its payload parses as. The +handle takes a CultCache document definition, and its `latest()` result is read +and parsed through that definition: a payload is a `DocumentPayload`, which only `decodeDocumentPayload(definition, payload)` reads, so no document is read without the definition that says how its bytes are laid out. @@ -421,8 +435,8 @@ Use `node.syncDocumentFromPeerSnapshot(...)` or `CultMesh.syncDocumentFromPeerSnapshot(...)` when a runtime should hydrate its local node from the remote snapshot. The helper requests the raw snapshot, applies it through the node's document registry, and returns the requested -typed definition. Same-schema aliases stay local after the call: the caller can -keep using `node.document(aliasDefinition, key)` and explicitly select +typed definition. The record then lives in the node: the caller can keep using +`node.document(definition, key)` and explicitly select `authoritativeWriter()`, `predictionWriter()`, or `observe()` without re-threading the RUDP peer. diff --git a/packages/cultmesh-ts/src/index.ts b/packages/cultmesh-ts/src/index.ts index 810e6919..90331655 100644 --- a/packages/cultmesh-ts/src/index.ts +++ b/packages/cultmesh-ts/src/index.ts @@ -3,7 +3,9 @@ import { decodeWireValue, encodeWireValue } from "cultnet-ts/wire"; import { CultCache, + SchemaConflictError, SingleFileMessagePackBackingStore, + schemaIdentityOf, type AnyCultCacheDocumentDefinition, type CacheBackingStore, type CultCacheEnvelope, @@ -207,6 +209,15 @@ export type CultMeshCollectionWatcher = ( callback: (change: CultMeshCollectionChange) => void, ) => CultMeshUnsubscribe; +/** + * The value type a catalog lookup yields. An explicit type argument wins; otherwise a CultCache document definition + * names its own value type, and a bare schema descriptor yields `unknown`. The value type is never inferred from a + * value or an expected result, so a wrong value is an error rather than a new type. + */ +export type CultMeshCatalogValue = [TDocument] extends [never] + ? TSchema extends AnyCultCacheDocumentDefinition ? CultCacheDocumentValue : unknown + : NoInfer; + export interface CultMeshDocumentSchemaDescriptor { readonly type?: string; readonly schemaId?: string; @@ -1019,38 +1030,6 @@ export class CultMeshDocumentHandle { ); } - public asSchemaAlias( - schema: CultMeshDocumentSchemaDescriptor, - options: { parse?: (value: unknown) => TAlias } = {}, - ): CultMeshDocumentHandle { - const aliasSchema = normalizeCultMeshDocumentSchema(schema); - if (!cultMeshSchemasAreCompatible(this.schema, aliasSchema)) { - throw new Error( - `Document '${this.documentId}' schema ${cultMeshSchemaLabel(this.schema)} is not compatible with alias ${cultMeshSchemaLabel(aliasSchema)}.`, - ); - } - - const parse = options.parse ?? ((value: unknown) => value as TAlias); - return new CultMeshDocumentHandle( - this.documentId, - aliasSchema, - async (context) => parse(await this.latest(context)), - { - sources: this.sources, - routeHint: this.routeHint, - watchDocument: this.watchDocument - ? (context, callback) => this.watch(context, value => callback(parse(value))) - : undefined, - replaceDocument: this.replaceDocument - ? (context, value) => this.replace(context, value as unknown as TDocument) - : undefined, - submitPrediction: this.submitPredictionDocument - ? (context, value) => this.submitPrediction(context, value as unknown as TDocument) - : undefined, - }, - ); - } - public bind(verse: CultMeshVerseContext | CultMeshVerse): CultMeshBoundDocumentHandle { return cultMeshBindDocument(verse, this); } @@ -1484,12 +1463,6 @@ export class CultMeshBoundDocumentHandle { }); } - public asSchemaAlias( - schema: CultMeshDocumentSchemaDescriptor, - options: { parse?: (value: unknown) => TAlias } = {}, - ): CultMeshBoundDocumentHandle { - return cultMeshBindDocument(this.verse, this.document.asSchemaAlias(schema, options)); - } } export class CultMeshDocumentCatalog { @@ -1509,6 +1482,7 @@ export class CultMeshDocumentCatalog { } public add(document: CultMeshDocumentHandle): this { + cultMeshRefuseSecondCatalogType(document.schema, this.#byType, this.#bySchemaId, this.#bySchemaNameVersion); this.#byDocumentId.set(document.documentId, document as CultMeshDocumentHandle); if (document.schema.type) { this.#byType.set(document.schema.type, document as CultMeshDocumentHandle); @@ -1523,129 +1497,120 @@ export class CultMeshDocumentCatalog { return this; } - public tryDocument( - schema: CultMeshDocumentSchemaDescriptor, - options: { parse?: (value: unknown) => TDocument } = {}, - ): CultMeshDocumentHandle | undefined { - const descriptor = normalizeCultMeshDocumentSchema(schema); - const exact = - (descriptor.type ? this.#byType.get(descriptor.type) : undefined) ?? - (descriptor.schemaId ? this.#bySchemaId.get(descriptor.schemaId) : undefined) ?? - (() => { - const key = cultMeshSchemaNameVersionKey(descriptor); - return key ? this.#bySchemaNameVersion.get(key) : undefined; - })(); - - if (!exact) { - return undefined; - } - - return cultMeshSchemasAreCompatible(exact.schema, descriptor) - ? exact.asSchemaAlias(descriptor, options) - : undefined; + // A catalog may hold several documents of one type, one per document id. A lookup by schema answers with one of them; + // a lookup by document id addresses each. + public tryDocumentById(documentId: string): CultMeshDocumentHandle | undefined { + return this.#byDocumentId.get(documentId); } - public document( - schema: CultMeshDocumentSchemaDescriptor, - options: { parse?: (value: unknown) => TDocument } = {}, - ): CultMeshDocumentHandle { - const document = this.tryDocument(schema, options); + public documentById(documentId: string): CultMeshDocumentHandle { + const document = this.tryDocumentById(documentId); if (!document) { - throw new Error(`Document catalog has no document for ${cultMeshSchemaLabel(schema)}.`); + throw new Error(`Document catalog has no document with id '${documentId}'.`); } return document; } - public latest( - schema: CultMeshDocumentSchemaDescriptor, + public tryDocument( + schema: TSchema, + ): CultMeshDocumentHandle> | undefined { + const descriptor = normalizeCultMeshDocumentSchema(schema); + // One type owns a schema, so a typed lookup is answered by that type's handle or not at all. + if (descriptor.type) { + return this.#byType.get(descriptor.type); + } + const key = cultMeshSchemaNameVersionKey(descriptor); + return (descriptor.schemaId ? this.#bySchemaId.get(descriptor.schemaId) : undefined) ?? + (key ? this.#bySchemaNameVersion.get(key) : undefined); + } + + public document( + schema: TSchema, + ): CultMeshDocumentHandle> { + return this.#require(schema); + } + + public latest( + schema: TSchema, context: CultMeshQueryContext | string = "local", - options: { parse?: (value: unknown) => TDocument } = {}, - ): Promise { - return this.document(schema, options).latest(context); + ): Promise> { + return this.#require(schema).latest(context); } public canReplace(schema: CultMeshDocumentSchemaDescriptor): boolean { return this.tryDocument(schema)?.canReplace ?? false; } - public replace( - schema: CultMeshDocumentSchemaDescriptor, - value: TDocument, - options?: { parse?: (value: unknown) => TDocument; context?: CultMeshQueryContext | string }, + public replace( + schema: TSchema, + value: CultMeshCatalogValue, + options?: { context?: CultMeshQueryContext | string }, ): Promise; - public replace( - schema: CultMeshDocumentSchemaDescriptor, + public replace( + schema: TSchema, context: CultMeshQueryContext | string, - value: TDocument, - options?: { parse?: (value: unknown) => TDocument }, + value: CultMeshCatalogValue, ): Promise; - public replace( + public replace( schema: CultMeshDocumentSchemaDescriptor, - contextOrValue: CultMeshQueryContext | string | TDocument, - valueOrOptions?: TDocument | { parse?: (value: unknown) => TDocument; context?: CultMeshQueryContext | string }, - maybeOptions: { parse?: (value: unknown) => TDocument } = {}, + contextOrValue: unknown, + valueOrOptions?: unknown, ): Promise { - const hasContext = - typeof contextOrValue === "string" || isCultMeshQueryContext(contextOrValue); - const context = hasContext - ? contextOrValue as CultMeshQueryContext | string - : (valueOrOptions as { context?: CultMeshQueryContext | string } | undefined)?.context ?? "local"; - const value = hasContext ? valueOrOptions as TDocument : contextOrValue as TDocument; - const options = hasContext ? maybeOptions : valueOrOptions as { parse?: (value: unknown) => TDocument } | undefined; - return this.document(schema, options).replace(context, value); + const [context, value] = cultMeshCatalogWriteArguments(contextOrValue, valueOrOptions); + return this.#require(schema).replace(context, value); } public canSubmitPrediction(schema: CultMeshDocumentSchemaDescriptor): boolean { return this.tryDocument(schema)?.canSubmitPrediction ?? false; } - public submitPrediction( - schema: CultMeshDocumentSchemaDescriptor, - value: TDocument, - options?: { parse?: (value: unknown) => TDocument; context?: CultMeshQueryContext | string }, + public submitPrediction( + schema: TSchema, + value: CultMeshCatalogValue, + options?: { context?: CultMeshQueryContext | string }, ): Promise; - public submitPrediction( - schema: CultMeshDocumentSchemaDescriptor, + public submitPrediction( + schema: TSchema, context: CultMeshQueryContext | string, - value: TDocument, - options?: { parse?: (value: unknown) => TDocument }, + value: CultMeshCatalogValue, ): Promise; - public submitPrediction( + public submitPrediction( schema: CultMeshDocumentSchemaDescriptor, - contextOrValue: CultMeshQueryContext | string | TDocument, - valueOrOptions?: TDocument | { parse?: (value: unknown) => TDocument; context?: CultMeshQueryContext | string }, - maybeOptions: { parse?: (value: unknown) => TDocument } = {}, + contextOrValue: unknown, + valueOrOptions?: unknown, ): Promise { - const hasContext = - typeof contextOrValue === "string" || isCultMeshQueryContext(contextOrValue); - const context = hasContext - ? contextOrValue as CultMeshQueryContext | string - : (valueOrOptions as { context?: CultMeshQueryContext | string } | undefined)?.context ?? "local"; - const value = hasContext ? valueOrOptions as TDocument : contextOrValue as TDocument; - const options = hasContext ? maybeOptions : valueOrOptions as { parse?: (value: unknown) => TDocument } | undefined; - return this.document(schema, options).submitPrediction(context, value); + const [context, value] = cultMeshCatalogWriteArguments(contextOrValue, valueOrOptions); + return this.#require(schema).submitPrediction(context, value); } - public authoritativeWriter( - schema: CultMeshDocumentSchemaDescriptor, - options: { parse?: (value: unknown) => TDocument; context?: CultMeshQueryContext | string } = {}, - ): CultMeshDocumentWriter { - return this.document(schema, options).authoritativeWriter(options.context ?? "local"); + public authoritativeWriter( + schema: TSchema, + options: { context?: CultMeshQueryContext | string } = {}, + ): CultMeshDocumentWriter & object> { + return this.#require(schema).authoritativeWriter(options.context ?? "local"); } - public predictionWriter( - schema: CultMeshDocumentSchemaDescriptor, - options: { parse?: (value: unknown) => TDocument; context?: CultMeshQueryContext | string } = {}, - ): CultMeshDocumentWriter { - return this.document(schema, options).predictionWriter(options.context ?? "local"); + public predictionWriter( + schema: TSchema, + options: { context?: CultMeshQueryContext | string } = {}, + ): CultMeshDocumentWriter & object> { + return this.#require(schema).predictionWriter(options.context ?? "local"); } - public watch( - schema: CultMeshDocumentSchemaDescriptor, - callback: (value: TDocument) => void, - options: { parse?: (value: unknown) => TDocument; context?: CultMeshQueryContext | string } = {}, + public watch( + schema: TSchema, + callback: (value: CultMeshCatalogValue) => void, + options: { context?: CultMeshQueryContext | string } = {}, ): CultMeshUnsubscribe { - return this.document(schema, options).watch(options.context ?? "local", callback); + return this.#require(schema).watch(options.context ?? "local", callback); + } + + #require(schema: CultMeshDocumentSchemaDescriptor): CultMeshDocumentHandle { + const document = this.tryDocument(schema); + if (!document) { + throw new Error(`Document catalog has no document for ${cultMeshSchemaLabel(schema)}.`); + } + return document; } } @@ -1711,35 +1676,6 @@ export class CultMeshCollectionHandle { return this.watchCollection(this.resolveContext(context), callback); } - public asSchemaAlias( - schema: CultMeshDocumentSchemaDescriptor, - options: { parse?: (value: unknown) => TAlias } = {}, - ): CultMeshCollectionHandle { - const aliasSchema = normalizeCultMeshDocumentSchema(schema); - if (!cultMeshSchemasAreCompatible(this.schema, aliasSchema)) { - throw new Error( - `Collection '${this.collectionId}' schema ${cultMeshSchemaLabel(this.schema)} is not compatible with alias ${cultMeshSchemaLabel(aliasSchema)}.`, - ); - } - - const parse = options.parse ?? ((value: unknown) => value as TAlias); - return new CultMeshCollectionHandle( - this.collectionId, - aliasSchema, - async (context) => (await this.latest(context)).map(value => parse(value)), - { - sources: this.sources, - routeHint: this.routeHint, - watchCollection: this.watchCollection - ? (context, callback) => this.watchChanges(context, change => callback({ - ...change, - value: change.value === undefined ? undefined : parse(change.value), - })) - : undefined, - }, - ); - } - public bind(verse: CultMeshVerseContext | CultMeshVerse): CultMeshBoundCollectionHandle { return cultMeshBindCollection(verse, this); } @@ -1772,6 +1708,7 @@ export class CultMeshCollectionCatalog { } public add(collection: CultMeshCollectionHandle): this { + cultMeshRefuseSecondCatalogType(collection.schema, this.#byType, this.#bySchemaId, this.#bySchemaNameVersion); this.#byCollectionId.set(collection.collectionId, collection as CultMeshCollectionHandle); if (collection.schema.type) { this.#byType.set(collection.schema.type, collection as CultMeshCollectionHandle); @@ -1786,54 +1723,47 @@ export class CultMeshCollectionCatalog { return this; } - public tryCollection( - schema: CultMeshDocumentSchemaDescriptor, - options: { parse?: (value: unknown) => TDocument } = {}, - ): CultMeshCollectionHandle | undefined { + public tryCollection( + schema: TSchema, + ): CultMeshCollectionHandle> | undefined { const descriptor = normalizeCultMeshDocumentSchema(schema); - const exact = - (descriptor.type ? this.#byType.get(descriptor.type) : undefined) ?? - (descriptor.schemaId ? this.#bySchemaId.get(descriptor.schemaId) : undefined) ?? - (() => { - const key = cultMeshSchemaNameVersionKey(descriptor); - return key ? this.#bySchemaNameVersion.get(key) : undefined; - })(); - - if (!exact) { - return undefined; + // One type owns a schema, so a typed lookup is answered by that type's handle or not at all. + if (descriptor.type) { + return this.#byType.get(descriptor.type); } - - return cultMeshSchemasAreCompatible(exact.schema, descriptor) - ? exact.asSchemaAlias(descriptor, options) - : undefined; + const key = cultMeshSchemaNameVersionKey(descriptor); + return (descriptor.schemaId ? this.#bySchemaId.get(descriptor.schemaId) : undefined) ?? + (key ? this.#bySchemaNameVersion.get(key) : undefined); } - public collection( - schema: CultMeshDocumentSchemaDescriptor, - options: { parse?: (value: unknown) => TDocument } = {}, - ): CultMeshCollectionHandle { - const collection = this.tryCollection(schema, options); + public collection( + schema: TSchema, + ): CultMeshCollectionHandle> { + return this.#require(schema); + } + + public latest( + schema: TSchema, + context: CultMeshQueryContext | string = "local", + ): Promise>> { + return this.#require(schema).latest(context); + } + + public watchChanges( + schema: TSchema, + callback: (change: CultMeshCollectionChange>) => void, + options: { context?: CultMeshQueryContext | string } = {}, + ): CultMeshUnsubscribe { + return this.#require(schema).watchChanges(options.context ?? "local", callback); + } + + #require(schema: CultMeshDocumentSchemaDescriptor): CultMeshCollectionHandle { + const collection = this.tryCollection(schema); if (!collection) { throw new Error(`Collection catalog has no collection for ${cultMeshSchemaLabel(schema)}.`); } return collection; } - - public latest( - schema: CultMeshDocumentSchemaDescriptor, - context: CultMeshQueryContext | string = "local", - options: { parse?: (value: unknown) => TDocument } = {}, - ): Promise> { - return this.collection(schema, options).latest(context); - } - - public watchChanges( - schema: CultMeshDocumentSchemaDescriptor, - callback: (change: CultMeshCollectionChange) => void, - options: { parse?: (value: unknown) => TDocument; context?: CultMeshQueryContext | string } = {}, - ): CultMeshUnsubscribe { - return this.collection(schema, options).watchChanges(options.context ?? "local", callback); - } } export class CultMeshBoundCollectionHandle { @@ -1858,12 +1788,6 @@ export class CultMeshBoundCollectionHandle { return this.collection.watchChanges(cultMeshQueryContextFromVerse(this.verse), callback); } - public asSchemaAlias( - schema: CultMeshDocumentSchemaDescriptor, - options: { parse?: (value: unknown) => TAlias } = {}, - ): CultMeshBoundCollectionHandle { - return cultMeshBindCollection(this.verse, this.collection.asSchemaAlias(schema, options)); - } } export class CultMeshStatePointer { @@ -2415,6 +2339,17 @@ function resolveCultMeshVerseContext(verse: CultMeshVerseContext | CultMeshVerse return verse instanceof CultMeshVerse ? verse.context : verse; } +// A catalog write takes (value, options?) or (context, value); both resolve to one context and one value. +function cultMeshCatalogWriteArguments( + contextOrValue: unknown, + valueOrOptions: unknown, +): [CultMeshQueryContext | string, unknown] { + if (typeof contextOrValue === "string" || isCultMeshQueryContext(contextOrValue)) { + return [contextOrValue, valueOrOptions]; + } + return [(valueOrOptions as { context?: CultMeshQueryContext | string } | undefined)?.context ?? "local", contextOrValue]; +} + function isCultMeshQueryContext(value: unknown): value is CultMeshQueryContext { if (typeof value !== "object" || value === null) { return false; @@ -2672,19 +2607,11 @@ export function cultMeshDocumentFromPeerSnapshot> => { const record = await requestCultNetRawSnapshotDocument( peer, - resolvedSchemaId, + cultMeshAcceptedSchemaIds(definition), recordKey, { timeoutMs: options.timeoutMs, messageIdPrefix: options.messageIdPrefix ?? documentId, - accepts: candidate => { - try { - decodeDocumentPayload(definition, candidate.payload); - return true; - } catch { - return false; - } - }, }, ); return decodeDocumentPayload(definition, record.payload); @@ -3872,11 +3799,7 @@ export class CultMeshNode { definition: TDefinition, key: string, ): CultCacheDocumentValue | undefined { - const registered = this.resolveCacheDefinition(definition); - const value = this.cache.get(registered, key); - return value === undefined - ? undefined - : this.parseDocumentValue(definition, value); + return this.cache.get(definition, key); } public getRequired( @@ -3896,17 +3819,14 @@ export class CultMeshNode { key: string, value: CultCacheDocumentValue, ): Promise> { - const registered = this.resolveCacheDefinition(definition); - return this.cache - .put(registered, key, this.parseDocumentValue(definition, value) as never) - .then(stored => this.parseDocumentValue(definition, stored)); + return this.cache.put(definition, key, value); } public delete( definition: TDefinition, key: string, ): Promise { - return this.cache.delete(this.resolveCacheDefinition(definition), key); + return this.cache.delete(definition, key); } public document( @@ -3919,15 +3839,7 @@ export class CultMeshNode { } = {}, ): CultMeshDocumentHandle> { const documentId = options.documentId ?? `${definition.type}:${key}`; - return this.asRequestedDocument( - cultMeshDocumentFromCache( - this.cache, - this.resolveCacheDefinition(definition), - key, - { ...options, documentId }, - ), - definition, - ); + return cultMeshDocumentFromCache(this.cache, definition, key, { ...options, documentId }); } public async syncDocumentFromPeerSnapshot( @@ -3939,7 +3851,6 @@ export class CultMeshNode { messageIdPrefix?: string; } = {}, ): Promise> { - const registered = this.resolveCacheDefinition(definition); const schema = cultMeshSchemaFromDefinition(definition); const schemaId = schema.schemaId ?? schema.type; if (!schemaId) { @@ -3947,19 +3858,11 @@ export class CultMeshNode { } const document = await requestCultNetRawSnapshotDocument( peer, - schemaId, + [schemaId], key, { timeoutMs: options.timeoutMs, messageIdPrefix: options.messageIdPrefix ?? key, - accepts: record => { - try { - decodeDocumentPayload(definition, record.payload); - return true; - } catch { - return false; - } - }, }, ); await this.documents.applyRawDocumentPutMessage(this.cache, { @@ -3967,7 +3870,7 @@ export class CultMeshNode { messageId: options.messageIdPrefix ?? `sync:${key}`, document, }); - return this.parseDocumentValue(definition, this.cache.getRequired(registered, key)); + return this.cache.getRequired(definition, key); } public syncDocumentFromPublication( @@ -4004,14 +3907,7 @@ export class CultMeshNode { } = {}, ): CultMeshDocumentHandle> { const documentId = options.documentId ?? `${definition.type}:global`; - return this.asRequestedDocument( - cultMeshGlobalDocumentFromCache( - this.cache, - this.resolveCacheDefinition(definition), - { ...options, documentId }, - ), - definition, - ); + return cultMeshGlobalDocumentFromCache(this.cache, definition, { ...options, documentId }); } public collection( @@ -4023,46 +3919,12 @@ export class CultMeshNode { } = {}, ): CultMeshCollectionHandle> { const collectionId = options.collectionId ?? definition.type; - return cultMeshCollectionFromCache( - this.cache, - this.resolveCacheDefinition(definition), - { ...options, collectionId }, - ).asSchemaAlias>( - cultMeshSchemaFromDefinition(definition), - { parse: value => this.parseDocumentValue(definition, value) }, - ); + return cultMeshCollectionFromCache(this.cache, definition, { ...options, collectionId }); } public async flush(soft = false): Promise { await this.store.pushAll?.(this.cache.snapshot(), { soft }); } - - private resolveCacheDefinition( - definition: TDefinition, - ): AnyCultCacheDocumentDefinition { - const schema = cultMeshSchemaFromDefinition(definition); - return ( - this.documents.get(definition.type) ?? - (schema.schemaId ? this.documents.getBySchemaId(schema.schemaId) : undefined) - )?.definition ?? definition; - } - - private asRequestedDocument( - document: CultMeshDocumentHandle, - definition: TDefinition, - ): CultMeshDocumentHandle> { - return document.asSchemaAlias>( - cultMeshSchemaFromDefinition(definition), - { parse: value => this.parseDocumentValue(definition, value) }, - ); - } - - private parseDocumentValue( - definition: TDefinition, - value: unknown, - ): CultCacheDocumentValue { - return definition.schema.parse(value) as CultCacheDocumentValue; - } } export class CultMeshVerseCatalog { @@ -6281,12 +6143,8 @@ export class CultMesh { function cultMeshSchemaFromDefinition( definition: AnyCultCacheDocumentDefinition, ): CultMeshDocumentSchemaDescriptor { - return { - type: definition.type, - schemaId: definition.schemaId ?? definition.type, - schemaName: definition.schemaName, - schemaVersion: definition.schemaVersion, - }; + const { schemaId, schemaName, schemaVersion } = schemaIdentityOf(definition); + return { type: definition.type, schemaId, schemaName, schemaVersion }; } function normalizeCultMeshDocumentSchema( @@ -6308,6 +6166,32 @@ function normalizeCultMeshDocumentSchema( return normalized; } +// One type owns a schema in a catalog, as in a registry. A handle whose type differs from the one already holding its +// schema id or schema name and version, or whose type already holds another schema, is refused. Handles of one type +// for one schema may repeat (one per record), and a handle without a type claims no type. +function cultMeshRefuseSecondCatalogType( + schema: CultMeshDocumentSchemaDescriptor, + byType: ReadonlyMap, + bySchemaId: ReadonlyMap, + bySchemaNameVersion: ReadonlyMap, +): void { + if (!schema.type) { + return; + } + const key = cultMeshSchemaNameVersionKey(schema); + const sameType = byType.get(schema.type)?.schema; + const holder = + (sameType?.schemaId && schema.schemaId && sameType.schemaId !== schema.schemaId ? sameType : undefined) ?? + [schema.schemaId ? bySchemaId.get(schema.schemaId) : undefined, key ? bySchemaNameVersion.get(key) : undefined] + .map(held => held?.schema) + .find(held => held?.type !== undefined && held.type !== schema.type); + if (holder) { + const name = (held: CultMeshDocumentSchemaDescriptor) => held.schemaName ?? held.type ?? held.schemaId ?? ""; + throw new SchemaConflictError(schema.schemaId ?? schema.type, [name(holder), name(schema)], "", + `CultMesh catalog schema ${cultMeshSchemaLabel(schema)} is held by type "${holder.type}" and cannot also be held by type "${schema.type}".`); + } +} + function cultMeshSchemaNameVersionKey( schema: CultMeshDocumentSchemaDescriptor, ): string | undefined { @@ -6316,26 +6200,6 @@ function cultMeshSchemaNameVersionKey( : undefined; } -function cultMeshSchemasAreCompatible( - left: CultMeshDocumentSchemaDescriptor, - right: CultMeshDocumentSchemaDescriptor, -): boolean { - if (left.schemaId && right.schemaId && left.schemaId === right.schemaId) { - return true; - } - - if ( - left.schemaName && - right.schemaName && - left.schemaName === right.schemaName && - (left.schemaVersion ?? "") === (right.schemaVersion ?? "") - ) { - return true; - } - - return Boolean(left.type && right.type && left.type === right.type); -} - function inferCultMeshSchemaName(schemaId: string | undefined): string | undefined { if (!schemaId) { return undefined; @@ -6348,53 +6212,23 @@ function inferCultMeshSchemaName(schemaId: string | undefined): string | undefin return /^\d+$/.test(version) ? schemaId.slice(0, marker) : undefined; } +// The record a store document reads: the one under the handle's schema id, else one under an id the definition lists as +// compatible, as CultCache resolves it. A record under any other id is not this document, whatever +// its payload parses as. function resolveCultMeshStoreDocumentRecord( records: readonly CultCacheEnvelope[], schema: CultMeshDocumentSchemaDescriptor, definition: AnyCultCacheDocumentDefinition, ): CultCacheEnvelope | undefined { - const schemaId = schema.schemaId ?? schema.type; - return ( - records.find(candidate => schemaId !== undefined && candidate.schemaId === schemaId) ?? - records.find(candidate => - cultMeshEnvelopeMatchesSchema(candidate, schema) && cultMeshEnvelopeParsesAsDefinition(candidate, definition)) ?? - records.find(candidate => cultMeshEnvelopeParsesAsDefinition(candidate, definition)) - ); + const own = schema.schemaId ?? schema.type; + const accepted = [own, ...cultMeshAcceptedSchemaIds(definition)]; + return records.find(candidate => candidate.schemaId === own) + ?? records.find(candidate => candidate.schemaId !== undefined && accepted.includes(candidate.schemaId)); } -function cultMeshEnvelopeMatchesSchema( - envelope: CultCacheEnvelope, - schema: CultMeshDocumentSchemaDescriptor, -): boolean { - const entry = envelope.catalogEntry; - if (!entry) { - return false; - } - if ( - schema.schemaId && - (entry.schemaId === schema.schemaId || - envelope.schemaId === schema.schemaId || - entry.compatibleSchemaIds?.includes(schema.schemaId)) - ) { - return true; - } - return Boolean( - schema.schemaName && - entry.schemaName === schema.schemaName && - (schema.schemaVersion ?? "") === (entry.schemaVersion ?? ""), - ); -} - -function cultMeshEnvelopeParsesAsDefinition( - envelope: CultCacheEnvelope, - definition: AnyCultCacheDocumentDefinition, -): boolean { - try { - decodeDocumentPayload(definition, envelope.payload); - return true; - } catch { - return false; - } +// The ids a definition's records may be stored under: its own first, then the ids it declares compatible. +function cultMeshAcceptedSchemaIds(definition: AnyCultCacheDocumentDefinition): readonly string[] { + return schemaIdentityOf(definition).compatibleSchemaIds; } function cultMeshSchemaLabel(schema: CultMeshDocumentSchemaDescriptor): string { @@ -6441,14 +6275,15 @@ function normalizeRudpDocumentPut( }; } +// The peer's record at the key under the first schema id, else under another id in the list. A record at the key under any +// other id is another type's, and is never taken in its place, whatever its payload parses as; none is not-found. async function requestCultNetRawSnapshotDocument( peerOrProvider: CultNetPeer | (() => CultNetPeer | Promise), - schemaId: string, + schemaIds: readonly string[], recordKey: string, options: { timeoutMs?: number; messageIdPrefix?: string; - accepts?: (document: CultNetRawDocumentRecord) => boolean; } = {}, ): Promise { const peer = await resolveCultNetPeer(peerOrProvider); @@ -6457,13 +6292,11 @@ async function requestCultNetRawSnapshotDocument( messageId, timeoutMs: options.timeoutMs, }); - const candidates = response.documents.filter(candidate => candidate.recordKey === recordKey); - const document = - candidates.find(candidate => candidate.schemaId === schemaId) ?? - (options.accepts ? candidates.find(options.accepts) : undefined) ?? - candidates[0]; + const atKey = response.documents.filter(candidate => candidate.recordKey === recordKey); + const document = atKey.find(candidate => candidate.schemaId === schemaIds[0]) + ?? atKey.find(candidate => schemaIds.includes(candidate.schemaId)); if (!document) { - throw new Error(`CultMesh peer snapshot did not return ${schemaId} at ${recordKey}.`); + throw new Error(`CultMesh peer snapshot did not return ${schemaIds[0]} at ${recordKey}.`); } return document; } diff --git a/packages/cultmesh-ts/test/codec-documents.test.ts b/packages/cultmesh-ts/test/codec-documents.test.ts index 4843c09b..58c05568 100644 --- a/packages/cultmesh-ts/test/codec-documents.test.ts +++ b/packages/cultmesh-ts/test/codec-documents.test.ts @@ -15,13 +15,12 @@ import { documentPayloadFromBytes, encodeDocumentPayload, } from "@gamecult/cultcache-ts"; -import type { CacheBackingStore, CultCacheEnvelope } from "@gamecult/cultcache-ts"; import { CultNetDocumentRegistry, defineCultNetDocumentBinding, type CultNetPeer, } from "cultnet-ts"; -import { CultMesh, cultMeshDocumentFromSingleFile, cultMeshDocumentFromStore, type CultMeshRudpDocumentPut } from "../src/index"; +import { CultMesh, cultMeshDocumentFromSingleFile, type CultMeshRudpDocumentPut } from "../src/index"; // A document written in the reference's positional shape, and a plain map document beside it. Every mesh reader of a // payload reads it the way the cache does, so both read through the mesh. @@ -47,15 +46,6 @@ const codecReceipt = defineDocumentType({ }, "Tests.CodecReceipt"), }); -// The same record under another schema id: the store holds it as codec_note, so it is found by parsing, not by id. -const codecNoteAlias = defineDocumentType({ - type: "cultmesh.codec_note_alias", - schemaId: "cultmesh.codec_note_alias.v0", - schema: z.object({ noteId: z.string(), body: z.string() }), - name: "noteId", - codec: codecNote.codec, -}); - const plainNote = defineDocumentType({ type: "cultmesh.plain_note", schemaId: "cultmesh.plain_note.v0", @@ -73,17 +63,6 @@ test("the mesh reads a stored codec document as the cache does, and a plain docu await plainNode.put(plainNote, "n2", { noteId: "n2", body: "map" }); assert.deepEqual(await cultMeshDocumentFromSingleFile(codecPath, codecNote).latest(), { noteId: "n1", body: "positional" }); - assert.deepEqual(await cultMeshDocumentFromSingleFile(codecPath, codecNoteAlias).latest(), { noteId: "n1", body: "positional" }); - // A store holding a plain record and a codec record, the alias found by parsing takes the codec one whichever the - // single-file store (which sorts records by key) puts first: the probe has to refuse the plain record, not meet it - // second. - for (const [plainKey, codecKey] of [["a-plain", "z-codec"], ["z-plain", "a-codec"]] as const) { - const bothPath = join(dir, `both-${plainKey}.ccmp`); - const bothNode = await CultMesh.startNode(bothPath, { documents: [plainNote, codecNote] }); - await bothNode.put(plainNote, plainKey, { noteId: plainKey, body: "map" }); - await bothNode.put(codecNote, codecKey, { noteId: codecKey, body: "positional" }); - assert.deepEqual(await cultMeshDocumentFromSingleFile(bothPath, codecNoteAlias).latest(), { noteId: codecKey, body: "positional" }); - } assert.deepEqual(await cultMeshDocumentFromSingleFile(plainPath, plainNote).latest(), { noteId: "n2", body: "map" }); }); @@ -92,10 +71,6 @@ test("the mesh syncs a codec document from a peer snapshot and returns a codec r const dir = await mkdtemp(join(tmpdir(), "cultmesh-ts-codec-sync-")); const source = await CultMesh.startNode(join(dir, "source.ccmp"), { documents: [codecNote, plainNote] }); await source.put(codecNote, "n1", { noteId: "n1", body: "synced positional" }); - // Two records at one key, in two shapes: a peer snapshot handle for the alias schema (which neither holds) accepts the - // one that parses as the codec. - await source.put(plainNote, "n3", { noteId: "n3", body: "map" }); - await source.put(codecNote, "n3", { noteId: "n3", body: "positional" }); const target = await CultMesh.startNode(join(dir, "target.ccmp"), { documents: [codecNote] }); const commandBinding = defineCultNetDocumentBinding({ definition: codecNote }); const plainBinding = defineCultNetDocumentBinding({ definition: plainNote }); @@ -117,18 +92,12 @@ test("the mesh syncs a codec document from a peer snapshot and returns a codec r peer = await CultMesh.createRudpPeer("cultmesh-ts-codec-client", connectionId, `rudp://127.0.0.1:${server.bind.port}`, { resendDelayMs: 25, resendPollMs: 5, connectTimeoutMs: 1_000, }); - // The accepts probe reads the snapshot's payload as the codec: a plain decode would reject every codec snapshot. const synced = await target.syncDocumentFromPeerSnapshot(peer, codecNote, "n1", { timeoutMs: 1_000 }); assert.deepEqual(synced, { noteId: "n1", body: "synced positional" }); assert.deepEqual(target.getRequired(codecNote, "n1"), { noteId: "n1", body: "synced positional" }); - // A peer snapshot handle reads the same way, and accepts the snapshot by parsing it as the codec. + // A peer snapshot handle reads the same way. assert.deepEqual(await CultMesh.documentFromPeerSnapshot(peer, codecNote, "n1", { timeoutMs: 1_000 }).latest(), { noteId: "n1", body: "synced positional" }); - assert.deepEqual( - await CultMesh.documentFromPeerSnapshot(peer, codecNoteAlias, "n3", { timeoutMs: 1_000 }).latest(), - { noteId: "n3", body: "positional" }, - ); - const receipt = await CultMesh.publishRudpDocumentAndWaitForReceipt( "cultmesh-ts-codec-receipt", connectionId, `rudp://127.0.0.1:${server.bind.port}`, commandBinding, "n2", { noteId: "n2", body: "command" }, receiptBinding, @@ -192,27 +161,6 @@ test("a RUDP put naming a schema the server has no definition for reaches the ha } }); -test("a store read takes the record the catalog matches to the schema and the definition parses, not the first that merely parses", async () => { - const target = defineDocumentType({ type: "cultmesh.pick", schemaId: "cultmesh.pick.v1", schema: z.object({ n: z.number() }) }); - const lenient = defineDocumentType({ type: "cultmesh.pick", schemaId: "cultmesh.pick.v1", schema: { parse: (value: unknown) => value } }); - const entry = (schemaId: string, compatibleSchemaIds?: readonly string[]) => ({ - schemaId, schemaName: schemaId, schemaVersion: `${schemaId}.v1`, contentHash: "h", canonicalSchemaJson: "{}", compatibleSchemaIds, - }); - const envelope = (key: string, schemaId: string, value: unknown, compatibleSchemaIds?: readonly string[]): CultCacheEnvelope => ({ - key, type: "cultmesh.pick", schemaId, storedAt: "2026-10-11T00:00:00.000Z", - payload: encodeDocumentPayload(lenient, value), catalogEntry: entry(schemaId, compatibleSchemaIds), - }); - // None carries the definition's own schema id, so the exact-id find misses. The first parses and is foreign; the - // second is matched by the catalog and does not parse; the third is matched and parses. - const records = [ - envelope("a", "foreign.v1", { n: 1 }), - envelope("b", "alias-bad.v1", { n: "x" }, ["cultmesh.pick.v1"]), - envelope("c", "alias-good.v1", { n: 2 }, ["cultmesh.pick.v1"]), - ]; - const store: CacheBackingStore = { pullAll: async () => records, push: async () => undefined, delete: async () => undefined }; - assert.deepEqual(await cultMeshDocumentFromStore(store, target).latest(), { n: 2 }); -}); - // A payload nested past the limit: MAX_PAYLOAD_DEPTH + 1 one-element arrays around a nil. const pastLimitPayload = () => documentPayloadFromBytes(Uint8Array.from([...new Array(MAX_PAYLOAD_DEPTH + 1).fill(0x91), 0xc0])); diff --git a/packages/cultmesh-ts/test/cultmesh.test.ts b/packages/cultmesh-ts/test/cultmesh.test.ts index 611492a3..eccb4545 100644 --- a/packages/cultmesh-ts/test/cultmesh.test.ts +++ b/packages/cultmesh-ts/test/cultmesh.test.ts @@ -6,7 +6,7 @@ import test from "node:test"; import assert from "node:assert/strict"; import { decodeWireValue, encodeWireValue } from "cultnet-ts/wire"; import { z } from "zod"; -import { defineDocumentType, documentPayloadBytes, documentPayloadFromBytes } from "@gamecult/cultcache-ts"; +import { SchemaConflictError, defineDocumentType, documentPayloadBytes, documentPayloadFromBytes } from "@gamecult/cultcache-ts"; import { CultNetDocumentRegistry, CultNetPeer, @@ -35,7 +35,11 @@ const noteDocument = defineDocumentType({ name: "noteId", }); -const noteAliasDocument = defineDocumentType({ +type Note = z.infer; +type CultMeshDocumentSchemaDescriptorLike = { type?: string; schemaId?: string; schemaName?: string; schemaVersion?: string }; + +// Another type claiming noteDocument's schema. It is never registered beside noteDocument: one type owns a schema. +const otherTypeForNoteSchema = defineDocumentType({ type: "cultmesh.note.ui", schemaId: "cultmesh.note.v0", schema: z.object({ @@ -134,36 +138,142 @@ test("CultMesh TS document handles hide local cache plumbing behind typed reacti await waitFor(() => observed.includes("updated"), "document handle update"); unsubscribe(); - const alias = bound.asSchemaAlias(noteAliasDocument, { - parse: value => noteAliasDocument.schema.parse(value), - }); - assert.equal((await alias.latest()).body, "updated"); - const catalog = CultMesh.documents(document); - assert.equal(catalog.canReplace(noteAliasDocument), true); - await catalog.authoritativeWriter(noteAliasDocument, { - parse: value => noteAliasDocument.schema.parse(value), + assert.equal(catalog.canReplace(noteDocument), true); + await catalog.authoritativeWriter(noteDocument, { context: "browser-client", }).write({ noteId: "note:1", body: "catalog-updated", }); assert.equal( - (await catalog.latest(noteAliasDocument, "browser-client", { - parse: value => noteAliasDocument.schema.parse(value), - })).body, + (await catalog.latest(noteDocument, "browser-client")).body, "catalog-updated", ); + // A typed lookup is answered by its own type's handle; a handle of another type for the same schema is not it. + assert.equal(catalog.tryDocument(otherTypeForNoteSchema), undefined); + assert.equal(catalog.canReplace(otherTypeForNoteSchema), false); + assert.throws(() => catalog.document(otherTypeForNoteSchema), /no document for/); const incremented = await bound.authoritativeWriter().update(value => ({ ...value, body: `${value.body}:updated-through-bound-set`, })); assert.equal(incremented.body, "catalog-updated:updated-through-bound-set"); assert.equal((await document.latest()).body, "catalog-updated:updated-through-bound-set"); - assert.throws( - () => document.asSchemaAlias({ schemaId: "cultmesh.other.v0" }), - /not compatible/, - ); +}); + +test("CultMesh TS catalogs infer a lookup's value type from its document definition", async () => { + const documents = CultMesh.documents(CultMesh.document("note:typed", noteDocument, async () => ({ noteId: "note:typed", body: "typed" }))); + const collections = CultMesh.collections(CultMesh.collection("notes:typed", noteDocument, async () => [{ noteId: "note:typed", body: "typed" }])); + const inferred: Note = await documents.latest(noteDocument); + const inferredRows: readonly Note[] = await collections.latest(noteDocument); + assert.equal(inferred.body, "typed"); + assert.equal(inferredRows[0]?.body, "typed"); + // @ts-expect-error a definition names its value type: body is a string + const wrong: number = (await documents.latest(noteDocument)).body; + // @ts-expect-error a collection lookup infers its rows the same way + const wrongRows: readonly number[] = await collections.latest(noteDocument); + // @ts-expect-error a bare schema descriptor yields unknown + void (await documents.latest({ schemaId: "cultmesh.note.v0" })).body; + const explicit: { body: string } = await documents.latest<{ body: string }>({ schemaId: "cultmesh.note.v0" }); + assert.equal(explicit.body, "typed"); + void wrong; + void wrongRows; + + // Writes are checked against the definition's value type. Never called: the handle is read-only. + const typeOnly = () => { + // @ts-expect-error replace checks the value against the definition + void documents.replace(noteDocument, { noteId: "note:typed", body: 1 }); + // @ts-expect-error so does a prediction, with a context + void documents.submitPrediction(noteDocument, "local", { noteId: "note:typed" }); + documents.watch(noteDocument, value => { + // @ts-expect-error a watched value is the definition's value + const n: number = value.body; + void n; + }); + // @ts-expect-error a writer writes the definition's value + void documents.authoritativeWriter(noteDocument).write({ noteId: 1, body: "x" }); + }; + void typeOnly; +}); + +test("CultMesh TS catalogs hold one type per schema and refuse a second, typed", () => { + const sameNameVersion = { type: "cultmesh.note.named", schemaId: "sha256:note-named", schemaName: "cultmesh.note", schemaVersion: "cultmesh.note.v0" }; + const sameTypeOtherSchema = { type: "cultmesh.note", schemaId: "cultmesh.note.v9" }; + const note = { noteId: "note:held", body: "held" }; + const kinds = [ + { + catalog: (...schemas: CultMeshDocumentSchemaDescriptorLike[]) => + CultMesh.documents(...schemas.map((schema, index) => CultMesh.document(`doc:${index}`, schema, async () => note))), + handle: (schema: CultMeshDocumentSchemaDescriptorLike, id: string) => CultMesh.document(id, schema, async () => note), + find: (catalog: any, schema: CultMeshDocumentSchemaDescriptorLike) => catalog.document(schema).documentId, + }, + { + catalog: (...schemas: CultMeshDocumentSchemaDescriptorLike[]) => + CultMesh.collections(...schemas.map((schema, index) => CultMesh.collection(`doc:${index}`, schema, async () => [note]))), + handle: (schema: CultMeshDocumentSchemaDescriptorLike, id: string) => CultMesh.collection(id, schema, async () => [note]), + find: (catalog: any, schema: CultMeshDocumentSchemaDescriptorLike) => catalog.collection(schema).collectionId, + }, + ]; + for (const kind of kinds) { + for (const claimant of [otherTypeForNoteSchema, sameNameVersion, sameTypeOtherSchema]) { + for (const [first, second] of [[noteDocument, claimant], [claimant, noteDocument]] as const) { + const catalog = kind.catalog(first); + assert.throws(() => catalog.add(kind.handle(second, "doc:second") as any), (error: unknown) => { + assert.ok(error instanceof SchemaConflictError, String(error)); + assert.equal(error.schemaId, second.schemaId); + assert.deepEqual(error.schemaNames, ["cultmesh.note", "cultmesh.note"]); + assert.ok(error.message.includes(`type "${second.type}"`) && error.message.includes(`type "${first.type}"`), error.message); + assert.equal(error.recordKey, ""); + return true; + }); + assert.throws(() => kind.catalog(first, second), SchemaConflictError); + // The refused handle left nothing behind: the first type still answers. + assert.equal(kind.find(catalog, first), "doc:0"); + assert.equal(kind.find(catalog, { schemaId: first.schemaId }), "doc:0"); + } + } + // Handles of one type for one schema may repeat, the same handle may be added again, and an untyped handle claims no type. + const held = kind.handle(noteDocument, "doc:held"); + const catalog = kind.catalog(noteDocument); + catalog.add(held as any).add(held as any); + catalog.add(kind.handle({ schemaId: "cultmesh.note.v0" }, "doc:untyped") as any); + assert.equal(kind.find(catalog, noteDocument), "doc:held"); + } +}); + +test("CultMesh TS catalog writes take (value, options) or (context, value)", async () => { + const writes: string[] = []; + let current = { noteId: "note:writes", body: "initial" }; + const record = (kind: string) => async (context: { runtimeId: string }, value: Note) => { + writes.push(`${kind}:${context.runtimeId}:${value.body}`); + current = value; + }; + const catalog = CultMesh.documents(CultMesh.document("note:writes", noteDocument, async () => current, { + replaceDocument: record("replace"), + submitPrediction: record("predict"), + })); + const note = (body: string) => ({ noteId: "note:writes", body }); + + await catalog.replace(noteDocument, note("r1")); + await catalog.replace(noteDocument, note("r2"), {}); + await catalog.replace(noteDocument, note("r3"), { context: "pilot-a" }); + await catalog.replace(noteDocument, "pilot-b", note("r4")); + await catalog.replace(noteDocument, CultMesh.queryContext("pilot-c"), note("r5")); + await catalog.submitPrediction(noteDocument, note("p1")); + await catalog.submitPrediction(noteDocument, note("p2"), { context: "pilot-a" }); + await catalog.submitPrediction(noteDocument, "pilot-b", note("p3")); + + assert.deepEqual(writes, [ + "replace:local:r1", + "replace:local:r2", + "replace:pilot-a:r3", + "replace:pilot-b:r4", + "replace:pilot-c:r5", + "predict:local:p1", + "predict:pilot-a:p2", + "predict:pilot-b:p3", + ]); }); test("CultMesh TS document handles submit predictions through configured authority hooks", async () => { @@ -193,10 +303,10 @@ test("CultMesh TS document handles submit predictions through configured authori assert.equal(document.canReplace, false); assert.equal(document.canSubmitPrediction, true); assert.equal(bound.canSubmitPrediction, true); - assert.equal(catalog.canSubmitPrediction(noteAliasDocument), true); + assert.equal(catalog.canSubmitPrediction(noteDocument), true); + assert.equal(catalog.canSubmitPrediction(otherTypeForNoteSchema), false); - await catalog.predictionWriter(noteAliasDocument, { - parse: value => noteAliasDocument.schema.parse(value), + await catalog.predictionWriter(noteDocument, { context: "pilot-a", }).write({ noteId: "note:prediction", @@ -207,23 +317,19 @@ test("CultMesh TS document handles submit predictions through configured authori assert.deepEqual(predictions, ["predicted"]); assert.equal((await bound.latest()).body, "predicted"); - const alias = bound.asSchemaAlias(noteAliasDocument, { - parse: value => noteAliasDocument.schema.parse(value), - }); - assert.equal(alias.canSubmitPrediction, true); - await alias.predictionWriter().write({ + await bound.predictionWriter().write({ noteId: "note:prediction", - body: "alias-predicted", + body: "bound-predicted", }); - assert.deepEqual(predictions, ["predicted", "alias-predicted"]); - assert.equal((await document.latest("pilot-a")).body, "alias-predicted"); + assert.deepEqual(predictions, ["predicted", "bound-predicted"]); + assert.equal((await document.latest("pilot-a")).body, "bound-predicted"); const updated = await document.predictionWriter("pilot-a").update(value => ({ ...value, body: "updated-as-prediction", })); assert.equal(updated.body, "updated-as-prediction"); - assert.deepEqual(predictions, ["predicted", "alias-predicted", "updated-as-prediction"]); + assert.deepEqual(predictions, ["predicted", "bound-predicted", "updated-as-prediction"]); assert.throws( () => document.replace({ @@ -262,57 +368,42 @@ test("CultMesh TS documents with both capabilities require an immutable writer c assert.deepEqual(writes, ["authoritative:canonical", "prediction:predicted"]); }); -test("CultMesh TS store document handles choose compatible foreign schema payloads", async () => { - const store = { - async pullAll() { - return [ - { - key: "note:foreign", - type: "runtime.generated.incompatible-note", - schemaId: "runtime.generated.incompatible-note.v1", - storedAt: "2026-06-27T00:00:00Z", - payload: payloadOf({ nope: "not the requested shape" }), - catalogEntry: { - schemaId: "runtime.generated.incompatible-note.v1", - schemaName: "runtime.generated.incompatible-note", - schemaVersion: "runtime.generated.incompatible-note.v1", - contentHash: "runtime.generated.incompatible-note.v1", - canonicalSchemaJson: "{}", - compatibleSchemaIds: ["runtime.generated.incompatible-note.v1"], - }, - }, - { - key: "note:foreign", - type: "runtime.generated.cultmesh.note", - schemaId: "runtime.generated.cultmesh.note.ui.42", - storedAt: "2026-06-27T00:00:01Z", - payload: payloadOf({ - noteId: "note:foreign", - body: "foreign schema id, local store shape", - }), - catalogEntry: { - schemaId: "runtime.generated.cultmesh.note.ui.42", - schemaName: "cultmesh.note", - schemaVersion: "cultmesh.note.v1", - contentHash: "runtime.generated.cultmesh.note.ui.42", - canonicalSchemaJson: "{}", - compatibleSchemaIds: ["runtime.generated.cultmesh.note.ui.42"], - }, - }, - ]; - }, +// A store document reads the record under its own schema id, or under an id its definition lists as compatible. A record +// under any other id is another type's, and is never read in its place because its payload happens to parse. +test("CultMesh TS store document handles read only records under an id their definition declares", async () => { + const envelope = (schemaId: string, value: unknown) => ({ + key: "note:store", + type: schemaId, + schemaId, + storedAt: "2026-06-27T00:00:00Z", + payload: payloadOf(value), + }); + const storeOf = (...records: ReturnType[]) => ({ + async pullAll() { return records; }, async push() {}, async delete() {}, - }; - - const document = CultMesh.documentFromStore(store, noteAliasDocument, { - documentId: "note:foreign.store", }); + const foreign = envelope("runtime.generated.cultmesh.note.ui.42", { noteId: "note:store", body: "foreign id, local shape" }); - assert.deepEqual(await document.latest(), { - noteId: "note:foreign", - body: "foreign schema id, local store shape", + await assert.rejects( + CultMesh.documentFromStore(storeOf(foreign), noteDocument).latest(), + /did not contain schema cultmesh\.note\.v0/, + ); + + const own = envelope("cultmesh.note.v0", { noteId: "note:store", body: "own id" }); + assert.equal((await CultMesh.documentFromStore(storeOf(foreign, own), noteDocument).latest()).body, "own id"); + + const renamed = defineDocumentType({ + type: "cultmesh.note-renamed", + schemaId: "cultmesh.note-renamed.v1", + compatibleSchemaIds: ["cultmesh.note.v0"], + schema: noteDocument.schema, }); + assert.equal((await CultMesh.documentFromStore(storeOf(foreign, own), renamed).latest()).body, "own id"); + await assert.rejects(CultMesh.documentFromStore(storeOf(foreign), renamed).latest(), /did not contain schema/); + // A record under the definition's own id wins over one under an id it lists, wherever each sits in the store. + const renamedOwn = envelope("cultmesh.note-renamed.v1", { noteId: "note:store", body: "renamed own id" }); + assert.equal((await CultMesh.documentFromStore(storeOf(own, renamedOwn), renamed).latest()).body, "renamed own id"); }); test("CultMesh TS reactive documents submit predictions from explicit updates", async () => { @@ -401,7 +492,7 @@ test("CultMesh TS reactive updates publish nested edits while snapshot mutation reactive.dispose(); }); -test("CultMesh TS node opens same-schema aliases with one call", async () => { +test("CultMesh TS node opens its registered document with one call", async () => { const filePath = join(await mkdtemp(join(tmpdir(), "cultmesh-ts-node-reactive-")), "node.ccmp"); const node = await CultMesh.startNode(filePath, { documents: [noteDocument], @@ -411,34 +502,25 @@ test("CultMesh TS node opens same-schema aliases with one call", async () => { body: "initial", }); - const aliasHandle = node.document(noteAliasDocument, "note:node-reactive", { + const handle = node.document(noteDocument, "note:node-reactive", { pollMs: 5, }); - assert.equal(aliasHandle.documentId, "cultmesh.note.ui:note:node-reactive"); - assert.equal((await aliasHandle.latest()).body, "initial"); - await aliasHandle.authoritativeWriter().write({ + assert.equal(handle.documentId, "cultmesh.note:note:node-reactive"); + assert.equal((await handle.latest()).body, "initial"); + await handle.authoritativeWriter().write({ noteId: "note:node-reactive", - body: "edited-through-alias-handle", - }); - assert.equal( - node.getRequired(noteAliasDocument, "note:node-reactive").body, - "edited-through-alias-handle", - ); - - await node.put(noteAliasDocument, "note:node-reactive", { - noteId: "note:node-reactive", - body: "edited-through-alias-put", + body: "edited-through-handle", }); assert.equal( node.getRequired(noteDocument, "note:node-reactive").body, - "edited-through-alias-put", + "edited-through-handle", ); assert.deepEqual( - (await node.collection(noteAliasDocument).latest()).map(note => note.body), - ["edited-through-alias-put"], + (await node.collection(noteDocument).latest()).map(note => note.body), + ["edited-through-handle"], ); - const reactive = aliasHandle.authoritativeWriter( + const reactive = handle.authoritativeWriter( CultMesh.queryContext("browser-client"), ).reactive({ watch: false, @@ -449,18 +531,12 @@ test("CultMesh TS node opens same-schema aliases with one call", async () => { }); await waitFor( () => node.getRequired(noteDocument, "note:node-reactive").body === "edited-through-node-helper", - "node-level reactive alias write", + "node-level reactive write", ); assert.equal(reactive.snapshot.noteId, "note:node-reactive"); assert.equal( - node.document(noteDocument, "note:node-reactive").asSchemaAlias(noteAliasDocument, { - parse: value => noteAliasDocument.schema.parse(value), - }).canReplace, - true, - ); - assert.equal( - (await node.document(noteAliasDocument, "note:node-reactive").observe({ + (await node.document(noteDocument, "note:node-reactive").observe({ watch: false, }).ready).body, "edited-through-node-helper", @@ -468,6 +544,33 @@ test("CultMesh TS node opens same-schema aliases with one call", async () => { reactive.dispose(); }); +test("CultMesh TS node refuses a type it did not register, even one claiming a registered schema", async () => { + const filePath = join(await mkdtemp(join(tmpdir(), "cultmesh-ts-node-other-type-")), "node.ccmp"); + const node = await CultMesh.startNode(filePath, { + documents: [noteDocument], + }); + await node.put(noteDocument, "note:owned", { noteId: "note:owned", body: "owned" }); + const notRegistered = /"cultmesh\.note\.ui" is not registered/; + + assert.throws(() => node.get(otherTypeForNoteSchema, "note:owned"), notRegistered); + await assert.rejects( + node.put(otherTypeForNoteSchema, "note:owned", { noteId: "note:owned", body: "other" }), + notRegistered, + ); + await assert.rejects(node.delete(otherTypeForNoteSchema, "note:owned"), notRegistered); + await assert.rejects(node.document(otherTypeForNoteSchema, "note:owned").latest(), notRegistered); + await assert.rejects(node.collection(otherTypeForNoteSchema).latest(), notRegistered); + assert.equal(node.getRequired(noteDocument, "note:owned").body, "owned"); + + // The node cannot come to hold the other type beside its owner either. + await assert.rejects( + CultMesh.startNode(join(await mkdtemp(join(tmpdir(), "cultmesh-ts-node-two-types-")), "node.ccmp"), { + documents: [noteDocument, otherTypeForNoteSchema], + }), + (error: unknown) => error instanceof SchemaConflictError && error.schemaId === "cultmesh.note.v0", + ); +}); + test("CultMesh TS reactive documents store reconciliation deltas after misprediction", async () => { let current = { noteId: "note:reconcile", @@ -949,15 +1052,13 @@ test("CultMesh TS binds publication document catalogs from source resolvers", as ["daemon:first", "daemon:second"], ); assert.equal( - (await catalog.document(noteAliasDocument, { - parse: value => noteAliasDocument.schema.parse(value), - }).latest()).body, + (await catalog.document(noteDocument).latest()).body, "second source", ); assert.equal(catalog.document(noteDocument).routeHint.description, "publication catalog"); }); -test("CultMesh TS syncs configured publications into local node aliases", async () => { +test("CultMesh TS syncs configured publications into a local node", async () => { const sourcePath = join(await mkdtemp(join(tmpdir(), "cultmesh-ts-publication-sync-source-")), "source.ccmp"); const targetPath = join(await mkdtemp(join(tmpdir(), "cultmesh-ts-publication-sync-target-")), "target.ccmp"); const source = await CultMesh.startNode(sourcePath, { @@ -965,7 +1066,7 @@ test("CultMesh TS syncs configured publications into local node aliases", async }); await source.put(noteDocument, "note:published", { noteId: "note:published", - body: "publication source hydrates local alias", + body: "publication source hydrates the local node", }); await source.flush(); const target = await CultMesh.startNode(targetPath, { @@ -977,7 +1078,7 @@ test("CultMesh TS syncs configured publications into local node aliases", async kind: "single-file", path: sourcePath, }, - noteAliasDocument, + noteDocument, "note:published", ); const facadeSynced = await CultMesh.syncDocumentFromPublication( @@ -986,21 +1087,20 @@ test("CultMesh TS syncs configured publications into local node aliases", async kind: "single-file", path: sourcePath, }, - noteAliasDocument, + noteDocument, "note:published", ); assert.deepEqual(synced, { noteId: "note:published", - body: "publication source hydrates local alias", + body: "publication source hydrates the local node", }); assert.deepEqual(facadeSynced, synced); - assert.equal(target.getRequired(noteDocument, "note:published").body, "publication source hydrates local alias"); - assert.equal(target.getRequired(noteAliasDocument, "note:published").body, "publication source hydrates local alias"); - assert.equal((await target.document(noteAliasDocument, "note:published").observe({ watch: false }).ready).body, "publication source hydrates local alias"); + assert.equal(target.getRequired(noteDocument, "note:published").body, "publication source hydrates the local node"); + assert.equal((await target.document(noteDocument, "note:published").observe({ watch: false }).ready).body, "publication source hydrates the local node"); }); -test("CultMesh TS syncs configured publication catalogs into local node aliases", async () => { +test("CultMesh TS syncs configured publication catalogs into a local node", async () => { const firstPath = join(await mkdtemp(join(tmpdir(), "cultmesh-ts-publication-sync-catalog-first-")), "first.ccmp"); const secondPath = join(await mkdtemp(join(tmpdir(), "cultmesh-ts-publication-sync-catalog-second-")), "second.ccmp"); const targetPath = join(await mkdtemp(join(tmpdir(), "cultmesh-ts-publication-sync-catalog-target-")), "target.ccmp"); @@ -1027,7 +1127,7 @@ test("CultMesh TS syncs configured publication catalogs into local node aliases" CultMesh.publicationDocument(noteDocument, "note:first-sync", { documentId: "local:first", }), - CultMesh.publicationDocument(noteAliasDocument, "note:second-sync", { + CultMesh.publicationDocument(noteDocument, "note:second-sync", { documentId: "local:second", source: { kind: "single-file", @@ -1056,37 +1156,27 @@ test("CultMesh TS syncs configured publication catalogs into local node aliases" bindings, ); + // Two documents of one type, each addressed by its own id whatever order the catalog lists them in. assert.deepEqual( - catalog.documents.map(document => document.documentId), + catalog.documents.map(document => document.documentId).sort(), ["local:first", "local:second"], ); - assert.equal( - await catalog.document(noteDocument, { - parse: value => noteDocument.schema.parse(value), - }).latest().then(value => value.body), - "first synced publication", - ); - assert.equal( - await catalog.document(noteAliasDocument, { - parse: value => noteAliasDocument.schema.parse(value), - }).latest().then(value => value.body), - "second synced publication", - ); - assert.equal(catalog.document(noteAliasDocument).routeHint.description, "publication sync catalog"); + const secondSynced = catalog.documentById("local:second"); + assert.equal((await catalog.documentById("local:first").latest()).body, "first synced publication"); + assert.equal((await secondSynced.latest()).body, "second synced publication"); + assert.equal(secondSynced.documentId, "local:second"); + assert.equal(secondSynced.routeHint.description, "publication sync catalog"); assert.equal(target.getRequired(noteDocument, "note:second-sync").body, "second synced publication"); - assert.equal(target.getRequired(noteAliasDocument, "note:second-sync").body, "second synced publication"); - assert.equal( - await facadeCatalog.document(noteAliasDocument, { - parse: value => noteAliasDocument.schema.parse(value), - }).latest().then(value => value.body), - "second synced publication", - ); + assert.equal((await facadeCatalog.documentById("local:second").latest()).body, "second synced publication"); + assert.equal((await facadeCatalog.documentById("local:first").latest()).body, "first synced publication"); + assert.equal(catalog.tryDocumentById("note:second-sync"), undefined); + assert.throws(() => catalog.documentById("local:third"), /no document with id 'local:third'/); }); test("CultMesh TS document catalogs resolve semantic schema versions passed as schema ids", async () => { const current = { noteId: "note:semantic", - body: "semantic alias", + body: "semantic version", }; const document = CultMesh.document( "daemon:semantic-note", @@ -1097,18 +1187,18 @@ test("CultMesh TS document catalogs resolve semantic schema versions passed as s }, async () => current, { - routeHint: CultMesh.routeHint("shared-memory", "semantic alias catalog"), + routeHint: CultMesh.routeHint("shared-memory", "semantic version catalog"), }, ); const catalog = CultMesh.documents(document); - const alias = catalog.document({ + const bySchemaVersion = catalog.document({ schemaId: "cultmesh.note.v1", }); - assert.equal(alias.documentId, "daemon:semantic-note"); - assert.equal(alias.routeHint.description, "semantic alias catalog"); - assert.deepEqual(await alias.latest(), current); + assert.equal(bySchemaVersion.documentId, "daemon:semantic-note"); + assert.equal(bySchemaVersion.routeHint.description, "semantic version catalog"); + assert.deepEqual(await bySchemaVersion.latest(), current); }); test("CultMesh TS collection handles expose typed snapshots and reset watches", async () => { @@ -1138,19 +1228,14 @@ test("CultMesh TS collection handles expose typed snapshots and reset watches", await waitFor(() => changes.length >= 2, "collection reset after update"); unsubscribe(); - assert.deepEqual( - (await collection.asSchemaAlias(noteAliasDocument, { - parse: value => noteAliasDocument.schema.parse(value), - }).latest()).map(note => note.body).sort(), - ["alpha", "bravo"], - ); const catalog = CultMesh.collections(collection); assert.deepEqual( - (await catalog.latest(noteAliasDocument, "local", { - parse: value => noteAliasDocument.schema.parse(value), - })).map(note => note.body).sort(), + (await catalog.latest(noteDocument, "local")).map(note => note.body).sort(), ["alpha", "bravo"], ); + // A typed lookup is answered by its own type's handle; a handle of another type for the same schema is not it. + assert.equal(catalog.tryCollection(otherTypeForNoteSchema), undefined); + assert.throws(() => catalog.collection(otherTypeForNoteSchema), /no collection for/); assert.equal( catalog.collection({ schemaId: "cultmesh.note.v0" }).collectionId, collection.collectionId, @@ -1159,7 +1244,7 @@ test("CultMesh TS collection handles expose typed snapshots and reset watches", assert.ok(changes.every(kind => kind === "reset")); }); -test("CultMesh TS collection catalogs resolve same-schema alias watches", async () => { +test("CultMesh TS collection catalogs resolve watches by schema version", async () => { const current = [{ noteId: "note:catalog-watch", body: "initial", @@ -1185,27 +1270,25 @@ test("CultMesh TS collection catalogs resolve same-schema alias watches", async ); const catalog = CultMesh.collections(collection); const observed: string[] = []; - const semanticAlias = { schemaId: "cultmesh.note.v1" }; - const unsubscribe = catalog.watchChanges>(semanticAlias, change => { + const bySchemaVersion = { schemaId: "cultmesh.note.v1" }; + const unsubscribe = catalog.watchChanges>(bySchemaVersion, change => { if (change.value) { observed.push(change.value.body); } - }, { - parse: value => noteAliasDocument.schema.parse(value), }); watcher?.({ kind: "updated", value: { noteId: "note:catalog-watch", - body: "through-catalog-alias", + body: "through-catalog", }, }); unsubscribe(); - assert.deepEqual(observed, ["through-catalog-alias"]); + assert.deepEqual(observed, ["through-catalog"]); assert.equal(watcher, undefined); - assert.equal(catalog.collection(semanticAlias).routeHint.description, "collection catalog"); + assert.equal(catalog.collection(bySchemaVersion).routeHint.description, "collection catalog"); }); test("CultMesh TS local authority leases do not trust peer cards by contact alone", () => { @@ -3021,33 +3104,31 @@ test("CultMesh TS reads remote RUDP snapshots through document handles", async ( body: "remote handles feel local", }); assert.equal(document.schema.type, "cultmesh.note"); - - const alias = CultMesh.documentFromPeerSnapshot( - peer, - defineDocumentType({ - type: "cultmesh.note.alias", - schemaId: "cultmesh.note.alias.v1", - schema: noteDocument.schema, - name: "noteId", - }), - "note:remote", - { - documentId: "note:remote.alias", - timeoutMs: 1_000, - }, - ); - - assert.deepEqual(await alias.latest(), { - noteId: "note:remote", - body: "remote handles feel local", - }); } finally { peer?.close(); server.close(); } }); -test("CultMesh TS syncs remote RUDP snapshots into a local node with same-schema aliases", async () => { +// What ships (F8): a node's reads and puts return the object its cache holds, not a copy, and the README says that object +// belongs to the cache. The caller's own value is never the one held. +test("CultMesh TS node reads and puts return the cache's own object", async () => { + const node = await CultMesh.startNode( + join(await mkdtemp(join(tmpdir(), "cultmesh-ts-held-object-")), "node.ccmp"), + { documents: [noteDocument] }, + ); + const written = { noteId: "note:held", body: "held" }; + const put = await node.put(noteDocument, "note:held", written); + assert.notEqual(put, written); + assert.deepEqual(put, written); + assert.equal(node.get(noteDocument, "note:held"), put); + assert.equal(node.getRequired(noteDocument, "note:held"), put); + assert.equal(node.get(noteDocument, "note:held"), node.get(noteDocument, "note:held")); + written.body = "changed by the caller"; + assert.equal(node.getRequired(noteDocument, "note:held").body, "held"); +}); + +test("CultMesh TS syncs remote RUDP snapshots into a local node", async () => { const connectionId = 0x1020304a; const source = await CultMesh.startNode( join(await mkdtemp(join(tmpdir(), "cultmesh-ts-rudp-sync-source-")), "node.ccmp"), @@ -3101,7 +3182,7 @@ test("CultMesh TS syncs remote RUDP snapshots into a local node with same-schema const synced = await target.syncDocumentFromPeerSnapshot( peer, - noteAliasDocument, + noteDocument, "note:sync-remote", { timeoutMs: 1_000, @@ -3110,7 +3191,7 @@ test("CultMesh TS syncs remote RUDP snapshots into a local node with same-schema const facadeSynced = await CultMesh.syncDocumentFromPeerSnapshot( target, peer, - noteAliasDocument, + noteDocument, "note:sync-remote", { timeoutMs: 1_000, @@ -3123,29 +3204,29 @@ test("CultMesh TS syncs remote RUDP snapshots into a local node with same-schema }); assert.deepEqual(facadeSynced, synced); assert.equal(target.getRequired(noteDocument, "note:sync-remote").body, "synced once, read as local"); - assert.equal(target.getRequired(noteAliasDocument, "note:sync-remote").body, "synced once, read as local"); - assert.equal((await target.document(noteAliasDocument, "note:sync-remote").latest()).body, "synced once, read as local"); + assert.equal((await target.document(noteDocument, "note:sync-remote").latest()).body, "synced once, read as local"); } finally { peer?.close(); server.close(); } }); -test("CultMesh TS peer snapshot handles choose compatible foreign schema payloads", async () => { +// A peer snapshot is read, and synced, only under the requested type's schema id. A record the peer holds at the key under +// another id is another type's: it is not-found for this type, and syncing never writes it anywhere. +test("CultMesh TS peer snapshots never take another type's record at the key", async () => { const connectionId = 0x10203049; - const node = await CultMesh.startNode( + const source = await CultMesh.startNode( join(await mkdtemp(join(tmpdir(), "cultmesh-ts-rudp-foreign-snapshot-")), "node.ccmp"), - { - documents: [incompatibleNoteDocument, foreignNoteDocument], - }, + { documents: [incompatibleNoteDocument, foreignNoteDocument] }, ); - await node.put(incompatibleNoteDocument, "note:foreign", { - nope: "not the requested shape", - }); - await node.put(foreignNoteDocument, "note:foreign", { - noteId: "note:foreign", - body: "foreign schema id, local shape", - }); + await source.put(incompatibleNoteDocument, "note:b", { nope: "remote b" }); + // Parses as a note: shape routing would store it as one. + await source.put(foreignNoteDocument, "note:c", { noteId: "note:c", body: "remote c, note-shaped" }); + const target = await CultMesh.startNode( + join(await mkdtemp(join(tmpdir(), "cultmesh-ts-rudp-foreign-target-")), "node.ccmp"), + { documents: [noteDocument, incompatibleNoteDocument] }, + ); + await target.put(incompatibleNoteDocument, "note:b", { nope: "local b" }); const server = CultMesh.createRudpDocumentServer( "cultmesh-ts-rudp-foreign-snapshot-server", @@ -3155,7 +3236,7 @@ test("CultMesh TS peer snapshot handles choose compatible foreign schema payload defineCultNetDocumentBinding({ definition: incompatibleNoteDocument }), defineCultNetDocumentBinding({ definition: foreignNoteDocument }), ]), - getCache: () => node.cache, + getCache: () => source.cache, bindHost: "127.0.0.1", bindPort: 0, resendDelayMs: 25, @@ -3181,20 +3262,22 @@ test("CultMesh TS peer snapshot handles choose compatible foreign schema payload }, ); - const alias = CultMesh.documentFromPeerSnapshot( - peer, - noteAliasDocument, - "note:foreign", - { - documentId: "note:foreign.alias", - timeoutMs: 1_000, - }, + for (const key of ["note:b", "note:c"]) { + const notFound = new RegExp(`did not return cultmesh\\.note\\.v0 at ${key}`); + await assert.rejects( + CultMesh.documentFromPeerSnapshot(peer, noteDocument, key, { timeoutMs: 1_000 }).latest(), + notFound, + ); + await assert.rejects(target.syncDocumentFromPeerSnapshot(peer, noteDocument, key, { timeoutMs: 1_000 }), notFound); + assert.equal(target.get(noteDocument, key), undefined, key); + } + // The failed sync of a note at "note:b" left the local record of the other type there untouched. + assert.deepEqual(target.getRequired(incompatibleNoteDocument, "note:b"), { nope: "local b" }); + // Its own type's record under its own id still syncs. + assert.deepEqual( + await target.syncDocumentFromPeerSnapshot(peer, incompatibleNoteDocument, "note:b", { timeoutMs: 1_000 }), + { nope: "remote b" }, ); - - assert.deepEqual(await alias.latest(), { - noteId: "note:foreign", - body: "foreign schema id, local shape", - }); } finally { peer?.close(); server.close(); diff --git a/packages/cultnet-py/CHANGELOG.md b/packages/cultnet-py/CHANGELOG.md new file mode 100644 index 00000000..cca85a79 --- /dev/null +++ b/packages/cultnet-py/CHANGELOG.md @@ -0,0 +1,9 @@ +# Changelog + +## Unreleased (breaking; version 0.x, so the next minor) + +- Removed `resolve_document_and_schema_id_for_raw_record` from the package exports. A raw record resolves to one local schema id, the one + its document registers, so there is no wire id to return. Use `resolve_document_for_raw_record(documents_by_schema_id, schema_id, record)`, + which returns the document; take the schema id from `document.catalog_entry().schema_id`. +- `apply_raw_document_record`, `apply_raw_snapshot` and `apply_shard_log_response` report and stamp the local schema id, also for a record + that arrived under a compatible id or the schema name. diff --git a/packages/cultnet-py/src/cultnet_py/__init__.py b/packages/cultnet-py/src/cultnet_py/__init__.py index e8b939c6..3fc36683 100644 --- a/packages/cultnet-py/src/cultnet_py/__init__.py +++ b/packages/cultnet-py/src/cultnet_py/__init__.py @@ -66,7 +66,7 @@ from .replication import ( apply_raw_document_record, apply_raw_snapshot, apply_shard_log_response, - resolve_document_and_schema_id_for_raw_record, + resolve_document_for_raw_record, schema_document_map, ) from .schema_catalog import ( @@ -228,7 +228,7 @@ __all__ = [ "apply_raw_document_record", "apply_raw_snapshot", "apply_shard_log_response", - "resolve_document_and_schema_id_for_raw_record", + "resolve_document_for_raw_record", "compute_simulation_claim_hash", "compute_reconnect_delay_ms", "create_reconnect_policy", diff --git a/packages/cultnet-py/src/cultnet_py/interop_peer.py b/packages/cultnet-py/src/cultnet_py/interop_peer.py index ba236686..e6cd4601 100644 --- a/packages/cultnet-py/src/cultnet_py/interop_peer.py +++ b/packages/cultnet-py/src/cultnet_py/interop_peer.py @@ -1316,6 +1316,7 @@ def witness_artifact_bundle_schema_json() -> str: "required": ["role", "schemaId"], "additionalProperties": False, "properties": { + "id": {"type": "string", "minLength": 1}, "role": {"type": "string", "minLength": 1}, "schemaId": {"type": "string", "minLength": 1}, "schemaVersion": {"type": "string", "minLength": 1}, @@ -1327,6 +1328,7 @@ def witness_artifact_bundle_schema_json() -> str: "required": ["role", "uri", "mediaType"], "additionalProperties": False, "properties": { + "id": {"type": "string", "minLength": 1}, "role": {"type": "string", "minLength": 1}, "uri": {"type": "string", "minLength": 1}, "mediaType": {"type": "string", "minLength": 1}, @@ -1340,6 +1342,7 @@ def witness_artifact_bundle_schema_json() -> str: "required": ["stage", "startedAt", "completedAt", "latencyMs"], "additionalProperties": False, "properties": { + "id": {"type": "string", "minLength": 1}, "stage": {"type": "string", "minLength": 1}, "startedAt": {"type": "string", "minLength": 1}, "completedAt": {"type": "string", "minLength": 1}, diff --git a/packages/cultnet-py/src/cultnet_py/replication.py b/packages/cultnet-py/src/cultnet_py/replication.py index 6450d4f9..531c0012 100644 --- a/packages/cultnet-py/src/cultnet_py/replication.py +++ b/packages/cultnet-py/src/cultnet_py/replication.py @@ -37,11 +37,9 @@ def apply_raw_document_record( ) -> CultNetAppliedRecord: wire = record.to_wire() if isinstance(record, CultNetRawDocumentRecord) else record schema_id = str(wire["schemaId"]) - document, resolved_schema_id = resolve_document_and_schema_id_for_raw_record( - documents_by_schema_id, - schema_id, - wire, - ) + document = resolve_document_for_raw_record(documents_by_schema_id, schema_id, wire) + # The record is applied, stamped and reported under the local schema id, as the cache stores it and a delete reports it. + resolved_schema_id = document.catalog_entry().schema_id envelope = raw_record_to_envelope(document, resolved_schema_id, wire) value = cache.put_envelope(document, envelope) return CultNetAppliedRecord( @@ -82,11 +80,8 @@ def apply_raw_snapshot( if not isinstance(record, dict): continue schema_id = str(record["schemaId"]) - document, resolved_schema_id = resolve_document_and_schema_id_for_raw_record( - documents_by_schema_id, - schema_id, - record, - ) + document = resolve_document_for_raw_record(documents_by_schema_id, schema_id, record) + resolved_schema_id = document.catalog_entry().schema_id envelope = raw_record_to_envelope(document, resolved_schema_id, record) batch = batches.setdefault(document.type, (document, [])) batch[1].append((resolved_schema_id, envelope)) @@ -159,27 +154,19 @@ def can_resolve_document_for_raw_record( return False +# routes a replicated raw record to a local binding; the store always persists the local schema id; this is not store-reader recovery. def resolve_document_for_raw_record( documents_by_schema_id: dict[str, DocumentDefinition[Any]], schema_id: str, record: dict[str, Any], ) -> DocumentDefinition[Any]: - return resolve_document_and_schema_id_for_raw_record(documents_by_schema_id, schema_id, record)[0] - - -def resolve_document_and_schema_id_for_raw_record( - documents_by_schema_id: dict[str, DocumentDefinition[Any]], - schema_id: str, - record: dict[str, Any], -) -> tuple[DocumentDefinition[Any], str]: document = documents_by_schema_id.get(schema_id) if document is not None: - return document, schema_id + return document schema_version = _infer_schema_version_from_payload(bytes(record["payload"])) schema_name = _infer_schema_name(schema_version) if schema_version is not None else None if schema_name is not None and schema_name in documents_by_schema_id: - document = documents_by_schema_id[schema_name] - return document, document.catalog_entry().schema_id + return documents_by_schema_id[schema_name] raise KeyError(schema_id) diff --git a/packages/cultnet-py/tests/test_cultnet.py b/packages/cultnet-py/tests/test_cultnet.py index 3c5793c1..19c88b6a 100644 --- a/packages/cultnet-py/tests/test_cultnet.py +++ b/packages/cultnet-py/tests/test_cultnet.py @@ -2102,6 +2102,51 @@ class CultNetTests(unittest.TestCase): self.assertEqual(applied_delete[0].change_kind, "removed") self.assertIsNone(cache.get(document, "policy:1")) + # A compatible schema id applies through the real replication function, the record is stamped with the registered id, and + # the store it leaves reopens and takes the next write. + def test_cultnet_raw_replication_under_a_compatible_schema_id_leaves_a_store_that_reopens(self) -> None: + import tempfile + from pathlib import Path + + from cultcache_py import SingleFileMessagePackBackingStore + + document = define_database_entry_type( + "replica.compatible-policy", + [("schema_version", 0), ("name", 1)], + schema_id="replica.compatible-policy.current", + schema_name="replica.compatible_policy", + schema_version="replica.compatible_policy.v1", + compatible_schema_ids=["replica.compatible-policy.current", "replica.compatible-policy.older"], + ) + value = {"schema_version": "replica.compatible_policy.v1", "name": "synced"} + record = { + "schemaId": "replica.compatible-policy.older", + "recordKey": "policy:1", + "storedAt": "2026-06-25T00:00:00Z", + "payloadEncoding": "messagepack", + "payload": document.encode_payload(value), + } + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "store.msgpack" + + def open_cache() -> CultCache: + cache = CultCache.builder().register_document_type(document).add_generic_store( + SingleFileMessagePackBackingStore(path) + ).build() + cache.pull_all_backing_stores() + return cache + + cache = open_cache() + applied = apply_raw_document_record(cache, schema_document_map([document]), record) + self.assertEqual(applied.schema_id, document.catalog_entry().schema_id) + self.assertEqual( + cache.get_required_envelope(document, "policy:1").schema_id, document.catalog_entry().schema_id + ) + reopened = open_cache() + self.assertEqual(reopened.get_required(document, "policy:1")["name"], "synced") + reopened.put(document, "policy:2", {**value, "name": "next"}) + self.assertEqual(open_cache().get_required(document, "policy:2")["name"], "next") + def test_cultnet_document_delete_helper_matches_schema_v0_shape(self) -> None: message = document_delete( message_id="delete-1", @@ -2425,9 +2470,9 @@ class CultNetTests(unittest.TestCase): witness_kind="interop-proof", captured_at="2026-06-13T00:00:03Z", subject={"documentType": "cultnet.interop-note", "subjectId": "note:python"}, - contracts=[{"role": "payload", "schemaId": "schema-a"}], - artifacts=[{"role": "log", "uri": "cultcache://bundle-1/log", "mediaType": "text/plain"}], - timing_witnesses=[{"stage": "roundtrip", "startedAt": "2026-06-13T00:00:03Z", "completedAt": "2026-06-13T00:00:04Z", "latencyMs": 1.0}], + contracts=[{"id": "c0e1a2b3c4d5", "role": "payload", "schemaId": "schema-a"}], + artifacts=[{"id": "a0e1a2b3c4d5", "role": "log", "uri": "cultcache://bundle-1/log", "mediaType": "text/plain"}], + timing_witnesses=[{"id": "e0e1a2b3c4d5", "stage": "roundtrip", "startedAt": "2026-06-13T00:00:03Z", "completedAt": "2026-06-13T00:00:04Z", "latencyMs": 1.0}], provenance={"pipelineId": "interop", "runId": "run-1", "runtimeId": "python-test"}, ) payload = encode_witness_artifact_bundle_payload(bundle) @@ -2443,6 +2488,8 @@ class CultNetTests(unittest.TestCase): self.assertEqual(slots[2], "2026-06-13T00:00:03Z") self.assertEqual(slots[3]["subjectId"], "note:python") self.assertEqual(slots[4][0]["schemaId"], "schema-a") + self.assertEqual([slots[4][0]["id"], slots[5][0]["id"], slots[6][0]["id"]], ["c0e1a2b3c4d5", "a0e1a2b3c4d5", "e0e1a2b3c4d5"]) + self.assertEqual(typed_decoded.contracts[0]["id"], "c0e1a2b3c4d5") self.assertEqual(slots[5][0]["uri"], "cultcache://bundle-1/log") self.assertEqual(slots[6][0]["latencyMs"], 1.0) self.assertEqual(slots[7]["runtimeId"], "python-test") @@ -2462,7 +2509,7 @@ class CultNetTests(unittest.TestCase): runtime_kind="python", display_name="Python Interop Test", agent_id="python-interop-test-agent", - schema_path=str(Path("contracts/cultnet/interop/cultnet.interop-note.schema.json")), + schema_path=str(Path(__file__).resolve().parents[3] / "contracts" / "cultnet" / "interop" / "cultnet.interop-note.schema.json"), ) binding = state.bindings["note"] logged_value = { diff --git a/packages/cultnet-rs/CHANGELOG.md b/packages/cultnet-rs/CHANGELOG.md new file mode 100644 index 00000000..d12dc3ed --- /dev/null +++ b/packages/cultnet-rs/CHANGELOG.md @@ -0,0 +1,12 @@ +# Changelog + +All notable changes to `cultnet-rs` are recorded here, following +[Keep a Changelog](https://keepachangelog.com/en/1.1.0/). Earlier releases are recorded +only in git history. + +## Unreleased + +### Breaking + +- `CultNetDocumentRegistry::register` returns `Result<&mut Self>` where it returned `&mut Self`. A binding that claims a document type or a schema id another binding already holds is refused with `cultcache_rs::SchemaConflictError` (naming the new binding's schema id and both document types) and nothing is bound; binding a type to the schema id it already holds still replaces that binding. Callers chain `register(...)?`. +- `CultMeshCdnChunkRef` gains a public `id` field, the decimal text of `offset` (the reference's element id, slot 4). A manifest written before ids reads `""`; encoding writes the slot. diff --git a/packages/cultnet-rs/examples/cultnet_interop_peer.rs b/packages/cultnet-rs/examples/cultnet_interop_peer.rs index a2a4847e..3d6700bf 100644 --- a/packages/cultnet-rs/examples/cultnet_interop_peer.rs +++ b/packages/cultnet-rs/examples/cultnet_interop_peer.rs @@ -262,7 +262,7 @@ fn serve(config: PeerConfig) -> Result<()> { cache.put(¬e.document_id, ¬e)?; let mut document_registry = CultNetDocumentRegistry::new(); - register_capability_bindings(&mut document_registry, &schema_registration.schema_id); + register_capability_bindings(&mut document_registry, &schema_registration.schema_id)?; let cache = Arc::new(Mutex::new(cache)); let document_registry = Arc::new(document_registry); @@ -384,7 +384,7 @@ fn dial(config: DialConfig) -> Result<()> { cache.pull_all_backing_stores()?; let mut document_registry = CultNetDocumentRegistry::new(); - register_capability_bindings(&mut document_registry, &schema_registration.schema_id); + register_capability_bindings(&mut document_registry, &schema_registration.schema_id)?; let transport_name = if config.target_rudp_port.is_some() { "rudp" @@ -1227,38 +1227,39 @@ fn register_capability_entry_types(cache: &mut CultCache) -> Result<()> { fn register_capability_bindings( document_registry: &mut CultNetDocumentRegistry, note_schema_id: &str, -) { +) -> Result<()> { document_registry .register(CultNetDocumentBinding::for_entry_with_schema_id::< CultNetInteropNote, >( note_schema_id.to_string(), INTEROP_SCHEMA_VERSION.to_string(), - )) + ))? .register(CultNetDocumentBinding::for_entry_with_schema_id::< CultNetInteropMutationIntent, >( MUTATION_INTENT_SCHEMA_ID.to_string(), MUTATION_INTENT_SCHEMA_VERSION.to_string(), - )) + ))? .register(CultNetDocumentBinding::for_entry_with_schema_id::< CultNetInteropMutationReceipt, >( MUTATION_RECEIPT_SCHEMA_ID.to_string(), MUTATION_RECEIPT_SCHEMA_VERSION.to_string(), - )) + ))? .register(CultNetDocumentBinding::for_entry_with_schema_id::< CultNetInteropFireCommand, >( FIRE_COMMAND_SCHEMA_ID.to_string(), FIRE_COMMAND_SCHEMA_VERSION.to_string(), - )) + ))? .register(CultNetDocumentBinding::for_entry_with_schema_id::< CultNetInteropFireReceipt, >( FIRE_RECEIPT_SCHEMA_ID.to_string(), FIRE_RECEIPT_SCHEMA_VERSION.to_string(), - )); + ))?; + Ok(()) } fn interaction_contracts() -> Vec { diff --git a/packages/cultnet-rs/src/content.rs b/packages/cultnet-rs/src/content.rs index 843d4b86..dbb58ff2 100644 --- a/packages/cultnet-rs/src/content.rs +++ b/packages/cultnet-rs/src/content.rs @@ -27,6 +27,8 @@ pub struct CultMeshCdnChunkRef { pub offset: i64, pub size_bytes: i32, pub record_key: String, + /// Element id (key 4): the decimal text of `offset`, never random. Absent in a manifest written before ids. + pub id: String, } /// `CultMeshCdnArtifactManifest`. Encodes as the reference's positional array (keys 0-9). @@ -50,7 +52,7 @@ pub struct CultMeshCdnArtifactManifest { // The tuple mirrors: `to_vec_named` would write a map for a named-field struct, but a tuple // struct is an array under every serde encoder. #[derive(Serialize, Deserialize)] -struct ChunkRefWire(String, i64, i32, String); +struct ChunkRefWire(String, i64, i32, String, #[serde(default)] String); #[derive(Serialize, Deserialize)] struct ManifestWire( @@ -105,7 +107,7 @@ mod ordered_map { impl From for ChunkRefWire { fn from(value: CultMeshCdnChunkRef) -> Self { - Self(value.chunk_hash, value.offset, value.size_bytes, value.record_key) + Self(value.chunk_hash, value.offset, value.size_bytes, value.record_key, value.id) } } @@ -116,6 +118,7 @@ impl From for CultMeshCdnChunkRef { offset: value.1, size_bytes: value.2, record_key: value.3, + id: value.4, } } } @@ -217,6 +220,7 @@ pub fn pack_content( offset, // A piece is at most `chunk_size` bytes, which fits an i32 (checked above). size_bytes: piece.len() as i32, + id: offset.to_string(), }); offset += piece.len() as i64; chunks.push(CultMeshCdnChunk { diff --git a/packages/cultnet-rs/src/replication.rs b/packages/cultnet-rs/src/replication.rs index d29714fe..743a28fd 100644 --- a/packages/cultnet-rs/src/replication.rs +++ b/packages/cultnet-rs/src/replication.rs @@ -3,6 +3,7 @@ use anyhow::anyhow; use cultcache_rs::CultCache; use cultcache_rs::CultCacheEnvelope; use cultcache_rs::DatabaseEntry; +use cultcache_rs::SchemaConflictError; use serde::Serialize; use serde::de::DeserializeOwned; use serde_json::Value; @@ -127,12 +128,31 @@ impl CultNetDocumentRegistry { Self::default() } - pub fn register(&mut self, binding: CultNetDocumentBinding) -> &mut Self { + /// Binds `binding`. One binding owns a document type and a schema id: a binding claiming a type or a schema id + /// another binding already holds is refused with [`SchemaConflictError`], naming the new binding's schema id and + /// both document types, and nothing is bound. Binding a type to the schema id it already holds replaces that binding. + pub fn register(&mut self, binding: CultNetDocumentBinding) -> Result<&mut Self> { + let holder = self + .bindings_by_type + .get(&binding.document_type) + .filter(|held| held.schema_id != binding.schema_id) + .or_else(|| { + self.bindings_by_schema_id + .get(&binding.schema_id) + .filter(|held| held.document_type != binding.document_type) + }); + if let Some(holder) = holder { + return Err(anyhow::Error::new(SchemaConflictError { + schema_id: binding.schema_id.clone(), + schema_names: vec![holder.document_type.clone(), binding.document_type.clone()], + record_key: String::new(), + })); + } self.bindings_by_schema_id .insert(binding.schema_id.clone(), binding.clone()); self.bindings_by_type .insert(binding.document_type.clone(), binding); - self + Ok(self) } pub fn binding(&self, document_type: &str) -> Option<&CultNetDocumentBinding> { diff --git a/packages/cultnet-rs/tests/content.rs b/packages/cultnet-rs/tests/content.rs index 178ebe3c..732d7d09 100644 --- a/packages/cultnet-rs/tests/content.rs +++ b/packages/cultnet-rs/tests/content.rs @@ -547,6 +547,7 @@ fn chunk_ref(hash: &str, offset: i64, size: i32) -> cultnet_rs::CultMeshCdnChunk offset, size_bytes: size, record_key: String::new(), + id: offset.to_string(), } } diff --git a/packages/cultnet-rs/tests/cultnet.rs b/packages/cultnet-rs/tests/cultnet.rs index df348cee..64c57aaf 100644 --- a/packages/cultnet-rs/tests/cultnet.rs +++ b/packages/cultnet-rs/tests/cultnet.rs @@ -1773,7 +1773,7 @@ fn document_registry_advertises_mutation_contracts_with_bindings() -> Result<()> "ghostlight.agent_state.v0".to_string(), ) .with_mutation_contract(contract.clone()), - ); + )?; assert_eq!(registry.mutation_contracts(), vec![contract]); Ok(()) @@ -1842,7 +1842,7 @@ fn document_registry_replicates_typed_cultcache_state() -> Result<()> { let mut registry = CultNetDocumentRegistry::new(); registry.register(CultNetDocumentBinding::for_entry::< GhostlightAgentStateFixture, - >("ghostlight.agent_state.v0".to_string())); + >("ghostlight.agent_state.v0".to_string()))?; let mut origin = CultCache::new(); origin.register_entry_type::()?; @@ -1899,7 +1899,7 @@ fn raw_snapshot_replication_preserves_messagepack_payload_bytes() -> Result<()> let mut registry = CultNetDocumentRegistry::new(); registry.register(CultNetDocumentBinding::for_entry::< GhostlightAgentStateFixture, - >("ghostlight.agent_state.v0".to_string())); + >("ghostlight.agent_state.v0".to_string()))?; let mut origin = CultCache::new(); origin.register_entry_type::()?; @@ -1945,7 +1945,7 @@ fn serve_read_only_raw_snapshot_omitted_v0_schema_list_is_no_filter_present_empt let mut registry = CultNetDocumentRegistry::new(); registry.register(CultNetDocumentBinding::for_entry_with_schema_id::< GhostlightAgentStateFixture, - >("ghostlight.agent-state".to_string(), None)); + >("ghostlight.agent-state".to_string(), None))?; let mut policy = CultNetReadOnlySnapshotPolicy::new(); policy.allow("ghostlight.agent-state", "row-1")?; @@ -2039,7 +2039,7 @@ fn serve_read_only_raw_snapshot_keeps_the_requested_record_key_order() -> Result let mut registry = CultNetDocumentRegistry::new(); registry.register(CultNetDocumentBinding::for_entry_with_schema_id::< GhostlightAgentStateFixture, - >("ghostlight.agent-state".to_string(), None)); + >("ghostlight.agent-state".to_string(), None))?; let mut policy = CultNetReadOnlySnapshotPolicy::new(); policy.allow("ghostlight.agent-state", "row-1")?; @@ -2099,10 +2099,64 @@ fn serve_read_only_raw_snapshot_keeps_the_requested_record_key_order() -> Result } #[test] -fn raw_snapshot_replication_hydrates_same_schema_rust_aliases() -> Result<()> { +fn document_registry_binds_one_type_per_schema_id_and_refuses_a_second_typed() -> Result<()> { + let canonical = |schema_id: &str| { + CultNetDocumentBinding::for_entry_with_schema_id::(schema_id.to_string(), None) + }; + let ui = |schema_id: &str| { + CultNetDocumentBinding::for_entry_with_schema_id::(schema_id.to_string(), None) + }; + let canonical_type = GhostlightAgentStateFixture::TYPE.to_string(); + let ui_type = GhostlightAgentStateUiFixture::TYPE.to_string(); + + // Another type for a held schema id, and a held type for another schema id, in both registration orders. + let cases = [ + (canonical("id.agent"), ui("id.agent"), "id.agent", [&canonical_type, &ui_type]), + (ui("id.agent"), canonical("id.agent"), "id.agent", [&ui_type, &canonical_type]), + (canonical("id.agent"), canonical("id.other"), "id.other", [&canonical_type, &canonical_type]), + (canonical("id.other"), canonical("id.agent"), "id.agent", [&canonical_type, &canonical_type]), + ]; + for (first, second, schema_id, names) in cases { + let mut registry = CultNetDocumentRegistry::new(); + registry.register(first.clone())?; + let error = registry.register(second.clone()).err().expect("a second claimant is refused"); + let conflict = error + .downcast_ref::() + .unwrap_or_else(|| panic!("{error:#}")); + assert_eq!(conflict.schema_id, schema_id); + assert_eq!(conflict.schema_names, names.map(|name| name.clone()).to_vec()); + assert_eq!(conflict.record_key, ""); + assert!(error.to_string().contains(schema_id), "{error:#}"); + // The refused binding left nothing behind: the first binding still answers for its type and its schema id. + assert_eq!(registry.binding(&first.document_type).map(|held| held.schema_id.as_str()), Some(first.schema_id.as_str())); + assert_eq!( + registry.binding_by_schema_id(&first.schema_id).map(|held| held.document_type.as_str()), + Some(first.document_type.as_str()) + ); + if second.schema_id != first.schema_id { + assert!(registry.binding_by_schema_id(&second.schema_id).is_none()); + } + } + + // A type bound again to the schema id it holds replaces its binding. + let mut registry = CultNetDocumentRegistry::new(); + registry.register(canonical("id.agent"))?; + registry.register(CultNetDocumentBinding::for_entry_with_schema_id::( + "id.agent".to_string(), + Some("ghostlight.agent_state.v2".to_string()), + ))?; + assert_eq!( + registry.binding(&canonical_type).and_then(|held| held.payload_schema_version.as_deref()), + Some("ghostlight.agent_state.v2") + ); + Ok(()) +} + +#[test] +fn raw_snapshot_replication_hydrates_the_receivers_own_type_for_the_schema() -> Result<()> { let temp = tempfile::tempdir()?; - let origin_store = temp.path().join("origin-alias.msgpack"); - let target_store = temp.path().join("target-alias.msgpack"); + let origin_store = temp.path().join("origin.msgpack"); + let target_store = temp.path().join("target.msgpack"); let canonical = GhostlightAgentStateFixture { schema_version: "ghostlight.agent_state.v0".to_string(), agent_id: "epiphany.persona".to_string(), @@ -2115,7 +2169,7 @@ fn raw_snapshot_replication_hydrates_same_schema_rust_aliases() -> Result<()> { >( "ghostlight.agent_state.v0".to_string(), "ghostlight.agent_state.v0".to_string(), - )); + ))?; let mut origin = CultCache::new(); origin.register_entry_type::()?; origin.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&origin_store))?; @@ -2123,24 +2177,24 @@ fn raw_snapshot_replication_hydrates_same_schema_rust_aliases() -> Result<()> { origin.put("epiphany.persona", &canonical)?; let raw_snapshot = origin_registry.create_raw_snapshot_response( &origin, - "raw-snapshot-alias", + "raw-snapshot-receiver-type", Some(&["ghostlight.agent_state.v0".to_string()]), Some(&["epiphany.persona".to_string()]), )?; - let mut alias_registry = CultNetDocumentRegistry::new(); - alias_registry.register(CultNetDocumentBinding::for_entry_with_schema_id::< + let mut target_registry = CultNetDocumentRegistry::new(); + target_registry.register(CultNetDocumentBinding::for_entry_with_schema_id::< GhostlightAgentStateUiFixture, >( "ghostlight.agent_state.v0".to_string(), "ghostlight.agent_state.v0".to_string(), - )); + ))?; let mut target = CultCache::new(); target.register_entry_type::()?; target.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&target_store))?; target.pull_all_backing_stores()?; - let applied = alias_registry + let applied = target_registry .sync_raw_document_from_snapshot_response::( &mut target, &raw_snapshot, @@ -2163,16 +2217,16 @@ fn raw_snapshot_replication_hydrates_same_schema_rust_aliases() -> Result<()> { } #[test] -fn reactive_document_coalesces_direct_same_schema_alias_member_writes() -> Result<()> { +fn reactive_document_coalesces_direct_member_writes() -> Result<()> { let temp = tempfile::tempdir()?; - let target_store = temp.path().join("target-reactive-alias.msgpack"); + let target_store = temp.path().join("target-reactive.msgpack"); let mut registry = CultNetDocumentRegistry::new(); registry.register(CultNetDocumentBinding::for_entry_with_schema_id::< GhostlightReactiveNoteUiFixture, >( "ghostlight.reactive_note.v0".to_string(), "ghostlight.reactive_note.v0".to_string(), - )); + ))?; let cache = Arc::new(Mutex::new(CultCache::new())); { let mut cache = cache.lock().expect("cache mutex"); @@ -2237,7 +2291,7 @@ fn reactive_document_tracks_canonical_reconciliation_delta() -> Result<()> { >( "ghostlight.reactive_note.v0".to_string(), "ghostlight.reactive_note.v0".to_string(), - )); + ))?; let cache = Arc::new(Mutex::new(CultCache::new())); { let mut cache = cache.lock().expect("cache mutex"); diff --git a/packages/cultnet-ts/CHANGELOG.md b/packages/cultnet-ts/CHANGELOG.md index 052d075a..d37ff6dc 100644 --- a/packages/cultnet-ts/CHANGELOG.md +++ b/packages/cultnet-ts/CHANGELOG.md @@ -3,6 +3,14 @@ All notable changes to `cultnet-ts` are recorded here, following [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). +## Unreleased + +### Breaking + +- `CultMeshCdnChunkRef` carries `id` (slot 4), the reference's element id. `encodeManifest` writes five-slot chunk references and `packContent` sets the decimal offset as the id, as the reference does; a manifest written before ids reads `""` and is rewritten with the slot. Content vectors regenerated from the reference. + +- Needs the cultcache-ts release that ships with this change (its manifest range moves with that release): the Idunn runtime authority reader calls `readSingleFileStore`, which 0.14.x does not export. An absent authority file is now a plain `Error` without a code. + ## [0.3.0] ### Added diff --git a/packages/cultnet-ts/README.md b/packages/cultnet-ts/README.md index ca0db88d..0075ed9d 100644 --- a/packages/cultnet-ts/README.md +++ b/packages/cultnet-ts/README.md @@ -143,7 +143,9 @@ imports no `node:` module, so a browser bundle can take it. - The decoder reads what MessagePack-CSharp reads and refuses what it refuses: nil in a string, tag or metadata slot is `null` and is written back as nil, a missing trailing slot takes the reference's default, extra slots and trailing - bytes are skipped, a float-encoded integer is refused, and invalid UTF-8 + bytes are skipped (a chunk reference's fifth slot is its element id, a string + or nil, `""` when absent; `packContent` writes the decimal offset as the id, as the + reference does), a float-encoded integer is refused, and invalid UTF-8 decodes to U+FFFD. A nil `chunks` is refused, as the reference's `ValidateManifest` refuses it. `content-acceptance-vectors.json` is the reference's own verdict on each of those shapes. diff --git a/packages/cultnet-ts/src/content.ts b/packages/cultnet-ts/src/content.ts index 6ff1e107..6f9fd873 100644 --- a/packages/cultnet-ts/src/content.ts +++ b/packages/cultnet-ts/src/content.ts @@ -26,6 +26,8 @@ export interface CultMeshCdnChunkRef { readonly offset: number; readonly sizeBytes: number; readonly recordKey: string | null; + // Element id, slot 4: the reference derives it from the offset when packing; "" when a manifest written before ids omits it. + readonly id: string | null; } // Metadata is ordered entries, never an object: a JS object reorders integer-like keys, and the entry order is wire bytes. @@ -142,13 +144,13 @@ const mapHeader = (count: number): Uint8Array => /** A string slot is written as C# writes it: a lone surrogate has no UTF-8 encoding and becomes U+FFFD at every length. */ const text = (field: T): T => (typeof field === "string" ? wellFormed(field) : field) as T; -/** The positional ten-array of the C# `[MessagePackObject]`: chunk references are four-arrays, metadata a map in entry order. */ +/** The positional ten-array of the C# `[MessagePackObject]`: chunk references are five-arrays, metadata a map in entry order. */ export function encodeManifest(manifest: CultMeshCdnArtifactManifest): Uint8Array { const { artifactId, kind, version, contentHash, sizeBytes, mimeType, createdAtUtc, chunks, tags, metadata } = manifest; return concat([ Uint8Array.of(0x9a), ...[artifactId, kind, version, contentHash, sizeBytes, mimeType, createdAtUtc].map(field => encode(text(field))), - encode(chunks.map(c => [text(c.chunkHash), c.offset, c.sizeBytes, text(c.recordKey)])), + encode(chunks.map(c => [text(c.chunkHash), c.offset, c.sizeBytes, text(c.recordKey), text(c.id)])), encode(tags === null ? null : tags.map(text)), ...(metadata === null ? [encode(null)] : [mapHeader(metadata.length), ...metadata.flatMap(([key, value]) => [encode(text(key)), encode(text(value))])]), ]); @@ -279,6 +281,7 @@ function readChunkRef(reader: Reader): CultMeshCdnChunkRef { // The reference's `int`. sizeBytes: slot(() => { const size = reader.integer("chunks.sizeBytes"); return size >= -(2 ** 31) && size < 2 ** 31 ? size : wrongType("chunks.sizeBytes"); }, 0), recordKey: slot(() => reader.string("chunks.recordKey"), ""), + id: slot(() => reader.string("chunks.id"), ""), }; for (; index < count; index += 1) reader.skip(); return chunk; @@ -308,7 +311,7 @@ export async function packContent( sizeBytes: bytes.length, mimeType: isNullOrWhiteSpaceCSharp(options.mimeType) ? "application/octet-stream" : options.mimeType!, createdAtUtc: options.createdAtUtc ?? `${new Date().toISOString().slice(0, 23)}0000+00:00`, - chunks: chunks.map(({ chunkHash, payload }, index) => ({ chunkHash, offset: offsets[index]!, sizeBytes: payload.length, recordKey: chunkRecordKey(chunkHash) })), + chunks: chunks.map(({ chunkHash, payload }, index) => ({ chunkHash, offset: offsets[index]!, sizeBytes: payload.length, recordKey: chunkRecordKey(chunkHash), id: String(offsets[index]) })), tags: options.tags ?? [], metadata: options.metadata ?? [], }; diff --git a/packages/cultnet-ts/src/idunn-runtime-authority.ts b/packages/cultnet-ts/src/idunn-runtime-authority.ts index 2ec5456c..0ca7f27b 100644 --- a/packages/cultnet-ts/src/idunn-runtime-authority.ts +++ b/packages/cultnet-ts/src/idunn-runtime-authority.ts @@ -1,7 +1,8 @@ import crypto from "node:crypto"; import fs from "node:fs"; import path from "node:path"; -import { documentPayloadFromBytes } from "@gamecult/cultcache-ts/document"; +import { readSingleFileStore } from "@gamecult/cultcache-ts"; +import { documentPayloadBytes, documentPayloadFromBytes } from "@gamecult/cultcache-ts/document"; import { decode, encode } from "@msgpack/msgpack"; import { @@ -28,7 +29,6 @@ export const IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT = "GAMECULT_IDUNN_CANDIDAT const PROVIDER_ID_DOMAIN = Buffer.from("gamecult.provider-health.identity.v1\0", "utf8"); const PROVIDER_PROTECTOR_CONTEXT = "gamecult-provider-health-identity-v1"; const ACTIVATION_ID_DOMAIN = Buffer.from("idunn.runtime-activation.id.v1\0", "utf8"); -const CULTCACHE_STORE_FORMAT = "cultcache.store.v1"; const ED25519_PKCS8_PREFIX = Buffer.from("302e020100300506032b657004220420", "hex"); // Limit: more than this many warmings signed between Idunn's observation and // assertWriteLease would push the leased one out and refuse a valid lease. @@ -319,25 +319,13 @@ function deepFreeze(value: T): T { } function readAuthorityRecord(filePath: string, type: string, schemaId: string): { payload: Uint8Array } { - const decoded = decode(fs.readFileSync(filePath)); - if (!Array.isArray(decoded) || decoded[0] !== CULTCACHE_STORE_FORMAT || !Array.isArray(decoded[1]) || !Array.isArray(decoded[2])) { - throw new Error(`Idunn authority file ${filePath} is not a CultCache v1 store.`); - } - if (decoded[2].length !== 1) throw new Error(`Idunn authority file ${filePath} must contain exactly one record.`); - const matchingRecords = (decoded[2] as unknown[]).filter((raw) => { - const row = array(raw, 4, "CultCache record"); - return row[1] === schemaId; - }); - if (matchingRecords.length !== 1) throw new Error(`Idunn authority file must contain exactly one ${schemaId} record.`); - const row = array(matchingRecords[0], 4, "CultCache record"); - const catalog = (decoded[1] as unknown[]).find((raw) => { - const entry = Array.isArray(raw) ? raw : []; - return entry[0] === schemaId; - }); - const catalogEntry = array(catalog, 7, "CultCache schema catalog entry"); - if (catalogEntry[1] !== type) throw new Error(`Idunn authority record has unexpected CultCache type ${String(catalogEntry[1])}.`); - if (!(row[3] instanceof Uint8Array)) throw new Error("Idunn authority payload is not MessagePack bytes."); - return { payload: row[3] }; + const store = readSingleFileStore(filePath); + if (store.absent) throw new Error(`Idunn authority file ${filePath} does not exist.`); + if (store.envelopes.length !== 1) throw new Error(`Idunn authority file ${filePath} must contain exactly one record.`); + const record = store.envelopes[0]!; + if (record.schemaId !== schemaId) throw new Error(`Idunn authority file must contain exactly one ${schemaId} record.`); + if (record.type !== type) throw new Error(`Idunn authority record has unexpected CultCache type ${record.type}.`); + return { payload: documentPayloadBytes(record.payload) }; } function decodeExpected(payload: Uint8Array): ExpectedIncarnation { diff --git a/packages/cultnet-ts/src/replication.ts b/packages/cultnet-ts/src/replication.ts index e12eaf8e..a332e4ba 100644 --- a/packages/cultnet-ts/src/replication.ts +++ b/packages/cultnet-ts/src/replication.ts @@ -1,15 +1,16 @@ import type { AnyCultCacheDocumentDefinition, CultCache, - CultCacheDocumentDefinition, CultCacheDocumentValue, CultCacheEnvelope, } from "@gamecult/cultcache-ts"; import { + SchemaConflictError, decodeDocumentPayload, documentPayloadBytes, documentPayloadFromBytes, encodeDocumentPayload, + schemaIdentityOf, } from "@gamecult/cultcache-ts"; import { @@ -42,7 +43,9 @@ export function defineCultNetDocumentBinding< export class CultNetDocumentRegistry { readonly #bindings = new Map(); - readonly #schemaBindings = new Map(); + // The binding that owns each schema id, and every binding listing an id as compatible, in registration order. + readonly #owners = new Map(); + readonly #listers = new Map(); constructor(bindings: Iterable = []) { for (const binding of bindings) { @@ -51,8 +54,32 @@ export class CultNetDocumentRegistry { } register(binding: CultNetDocumentBinding): this { + // One definition owns a type, a schema id and a schema name, derived as CultCache derives them. Another definition + // claiming any of them is refused, so a schema never resolves to two types. + const identity = schemaIdentityOf(binding.definition); + for (const holder of this.#bindings.values()) { + if (holder.definition === binding.definition) { + continue; + } + const held = schemaIdentityOf(holder.definition); + const claimed = + holder.definition.type === binding.definition.type ? `type "${binding.definition.type}"` + : held.schemaId === identity.schemaId ? `schema id "${identity.schemaId}"` + : held.schemaName === identity.schemaName ? `schema name "${identity.schemaName}"` + : undefined; + if (claimed) { + throw new SchemaConflictError(identity.schemaId, [held.schemaName, identity.schemaName], "", + `CultNet ${claimed} is already bound to type "${holder.definition.type}"; type "${binding.definition.type}" (schema id "${identity.schemaId}") cannot also be bound.`); + } + } this.#bindings.set(binding.definition.type, binding); - this.#schemaBindings.set(schemaIdForBinding(binding), binding); + this.#owners.set(identity.schemaId, binding); + for (const schemaId of identity.compatibleSchemaIds) { + const listers = this.#listers.get(schemaId) ?? []; + if (schemaId !== identity.schemaId && !listers.some(lister => lister.definition === binding.definition)) { + this.#listers.set(schemaId, [...listers, binding]); + } + } return this; } @@ -60,16 +87,22 @@ export class CultNetDocumentRegistry { return this.#bindings.get(documentType); } + // The binding a record under this schema id belongs to: the id's owner, else the one binding that lists it as + // compatible. An id nothing declares resolves to nothing. An id that several bindings list and none owns is refused, + // as the cache refuses it, whatever order the bindings were registered in. getBySchemaId(schemaId: string): CultNetDocumentBinding | undefined { - const exact = this.#schemaBindings.get(schemaId); - if (exact) { - return exact; + const owner = this.#owners.get(schemaId); + if (owner) { + return owner; } - - const requestedSchemaIds = new Set([schemaId]); - return this.#uniqueBindings().find(binding => - schemaMatchesBinding(schemaIdForBinding(binding), binding, requestedSchemaIds), - ); + const listers = this.#listers.get(schemaId) ?? []; + if (listers.length > 1) { + const types = listers.map(lister => lister.definition.type).sort(); + throw new SchemaConflictError(schemaId, listers.map(lister => schemaIdentityOf(lister.definition).schemaName).sort(), "", + `CultNet schema id "${schemaId}" is owned by no bound type and declared compatible by several (${types.join(", ")}), ` + + "so it resolves to none. Bind the type that owns the id, or declare it on one type."); + } + return listers[0]; } createDocumentPutMessage( @@ -169,7 +202,7 @@ export class CultNetDocumentRegistry { const binding = this.#requireBinding(envelope.type); const schemaId = schemaIdForEnvelope(envelope, binding); - if (requestedSchemaIds && !schemaMatchesBinding(schemaId, binding, requestedSchemaIds)) { + if (requestedSchemaIds && !answersTo(schemaId, binding, requestedSchemaIds)) { continue; } @@ -206,7 +239,7 @@ export class CultNetDocumentRegistry { const binding = this.#requireBinding(envelope.type); const schemaId = schemaIdForEnvelope(envelope, binding); - if (requestedSchemaIds && !schemaMatchesBinding(schemaId, binding, requestedSchemaIds)) { + if (requestedSchemaIds && !answersTo(schemaId, binding, requestedSchemaIds)) { continue; } @@ -274,16 +307,13 @@ export class CultNetDocumentRegistry { binding: CultNetDocumentBinding; envelope: CultCacheEnvelope; } { - const resolution = this.#resolveRawDocumentBinding(message.document); - const { binding } = resolution; + const binding = this.#requireSchemaBinding(message.document.schemaId); return { binding, envelope: { key: message.document.recordKey, type: binding.definition.type, - schemaId: resolution.preserveIncomingSchemaId - ? message.document.schemaId - : schemaIdForBinding(binding), + schemaId: schemaIdForBinding(binding), payload: documentPayloadFromBytes(new Uint8Array(documentPayloadBytes(message.document.payload))), storedAt: message.document.storedAt, }, @@ -334,35 +364,6 @@ export class CultNetDocumentRegistry { return binding; } - #resolveRawDocumentBinding(document: CultNetRawDocumentRecord): { - binding: CultNetDocumentBinding; - preserveIncomingSchemaId: boolean; - } { - const exact = this.getBySchemaId(document.schemaId); - if (exact) { - return { binding: exact, preserveIncomingSchemaId: true }; - } - - const payload = documentPayloadFromBytes(new Uint8Array(documentPayloadBytes(document.payload))); - for (const binding of this.#uniqueBindings()) { - try { - decodeDocumentPayload(binding.definition, payload); - return { binding, preserveIncomingSchemaId: false }; - } catch { - // Keep looking; a foreign schema id is acceptable only when the payload validates locally. - } - } - - throw new Error(`No CultNet document binding is registered for schema "${document.schemaId}".`); - } - - #uniqueBindings(): CultNetDocumentBinding[] { - return [...new Set([ - ...this.#schemaBindings.values(), - ...this.#bindings.values(), - ])]; - } - #createRawDocumentRecord(envelope: CultCacheEnvelope): CultNetRawDocumentRecord { const binding = this.#requireBinding(envelope.type); return { @@ -376,43 +377,17 @@ export class CultNetDocumentRegistry { } function schemaIdForBinding(binding: CultNetDocumentBinding): string { - return binding.definition.schemaId ?? binding.definition.type; + return schemaIdentityOf(binding.definition).schemaId; } -function schemaMatchesBinding( +// A held record matches a snapshot filter when the filter names the id it carries or any id its binding answers to. +function answersTo( schemaId: string, binding: CultNetDocumentBinding, requestedSchemaIds: Set, ): boolean { - if (requestedSchemaIds.has(schemaId)) { - return true; - } - - const definition = binding.definition; - if (definition.schemaId && requestedSchemaIds.has(definition.schemaId)) { - return true; - } - if (definition.schemaName && requestedSchemaIds.has(definition.schemaName)) { - return true; - } - if (definition.schemaVersion && requestedSchemaIds.has(definition.schemaVersion)) { - return true; - } - if (definition.compatibleSchemaIds?.some(candidate => requestedSchemaIds.has(candidate))) { - return true; - } - - return Array.from(requestedSchemaIds) - .some(candidate => inferSchemaName(candidate) === definition.schemaName); -} - -function inferSchemaName(schemaId: string): string | undefined { - const marker = schemaId.lastIndexOf(".v"); - if (marker <= 0 || marker + 2 >= schemaId.length) { - return undefined; - } - const version = schemaId.slice(marker + 2); - return /^\d+$/u.test(version) ? schemaId.slice(0, marker) : undefined; + return requestedSchemaIds.has(schemaId) + || schemaIdentityOf(binding.definition).compatibleSchemaIds.some(candidate => requestedSchemaIds.has(candidate)); } function schemaIdForEnvelope( diff --git a/packages/cultnet-ts/test/content.test.ts b/packages/cultnet-ts/test/content.test.ts index 3ff58d48..ec23b64e 100644 --- a/packages/cultnet-ts/test/content.test.ts +++ b/packages/cultnet-ts/test/content.test.ts @@ -49,14 +49,14 @@ const HASH = "a".repeat(64); const manifestOf = (over: Partial = {}): CultMeshCdnArtifactManifest => ({ artifactId: "id", kind: "asset", version: "1", contentHash: HASH, sizeBytes: 4, mimeType: "text/plain", createdAtUtc: "t", - chunks: [{ chunkHash: HASH, offset: 0, sizeBytes: 4, recordKey: `mesh:cdn:chunk:${HASH}` }], tags: ["t"], metadata: [["k", "v"]], + chunks: [{ chunkHash: HASH, offset: 0, sizeBytes: 4, recordKey: `mesh:cdn:chunk:${HASH}`, id: "0" }], tags: ["t"], metadata: [["k", "v"]], ...over, }); // Wire bytes of a ten-array with any field replaced, metadata as a plain object (no integer-like keys). const fieldsOf = (over: Record = {}) => { const m = manifestOf(); const fields: unknown[] = [m.artifactId, m.kind, m.version, m.contentHash, m.sizeBytes, m.mimeType, m.createdAtUtc, - m.chunks.map(c => [c.chunkHash, c.offset, c.sizeBytes, c.recordKey]), m.tags, { k: "v" }]; + m.chunks.map(c => [c.chunkHash, c.offset, c.sizeBytes, c.recordKey, c.id]), m.tags, { k: "v" }]; for (const [index, value] of Object.entries(over)) fields[Number(index)] = value; return fields; }; @@ -95,7 +95,7 @@ test("the packed chunks are copies that rejoin to the body, each named by the ha assert.deepEqual(Array.from(chunks.flatMap(c => Array.from(c.payload))), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); for (const [i, c] of chunks.entries()) { assert.equal(c.chunkHash, await sha256Hex(c.payload)); - assert.deepEqual(manifest.chunks[i], { chunkHash: c.chunkHash, offset: i * 4, sizeBytes: c.payload.length, recordKey: chunkRecordKey(c.chunkHash) }); + assert.deepEqual(manifest.chunks[i], { chunkHash: c.chunkHash, offset: i * 4, sizeBytes: c.payload.length, recordKey: chunkRecordKey(c.chunkHash), id: String(i * 4) }); } const twins = await packContent("x", new Uint8Array(8), { chunkSizeBytes: 4 }); assert.equal(twins.manifest.chunks[0]!.chunkHash, twins.manifest.chunks[1]!.chunkHash, "equal chunks share a hash"); @@ -216,7 +216,7 @@ test("a metadata key that is not a string names the field", () => { }); test("validateManifest refuses negative size, a gap, an overlap and a size-sum mismatch with the C# messages", () => { - const chunk = (offset: number, sizeBytes: number) => ({ chunkHash: HASH, offset, sizeBytes, recordKey: "k" }); + const chunk = (offset: number, sizeBytes: number) => ({ chunkHash: HASH, offset, sizeBytes, recordKey: "k", id: String(offset) }); const refuses = (over: Partial, message: string) => assert.throws(() => validateManifest(manifestOf(over)), new Error(message)); refuses({ sizeBytes: -1, chunks: [] }, "CDN artifact manifest has a negative size."); @@ -232,7 +232,7 @@ test("validateManifest refuses negative size, a gap, an overlap and a size-sum m }); test("validateManifest refuses a size that is not a whole number below 2^53 and a chunk size an int cannot hold, whether decoded or hand-built", () => { - const chunk = (offset: number, sizeBytes: number) => ({ chunkHash: HASH, offset, sizeBytes, recordKey: "k" }); + const chunk = (offset: number, sizeBytes: number) => ({ chunkHash: HASH, offset, sizeBytes, recordKey: "k", id: String(offset) }); const refuses = (over: Partial, message: string) => assert.throws(() => validateManifest(manifestOf(over)), new Error(message)); const manifestMessage = "CDN artifact manifest size is not a whole number below 2^53."; @@ -405,6 +405,23 @@ test("nil in a reference slot reads as null and writes back as nil; a missing tr assert.deepEqual(short, { artifactId: "id", kind: "asset", version: "", contentHash: "", sizeBytes: 0, mimeType: "application/octet-stream", createdAtUtc: "", chunks: [], tags: [], metadata: [] }); }); +test("a chunk reference's id is slot 4: nil reads null and writes nil, an absent slot reads \"\" and is written, packContent writes the decimal offset", async () => { + const chunk = (id: unknown[]) => [HASH, 0, 4, "k", ...id]; + const read = (id: unknown[]) => decodeManifest(encode(fieldsOf({ 7: [chunk(id)] }) as unknown[], { useBigInt64: true, maxDepth: 1000 })).chunks[0]!; + const nilled = encode(fieldsOf({ 7: [chunk([null])] }), { useBigInt64: true, maxDepth: 1000 }); + const nilManifest = decodeManifest(nilled); + assert.equal(nilManifest.chunks[0]!.id, null); + assert.deepEqual(encodeManifest(nilManifest), nilled); + const absent = decodeManifest(encode(fieldsOf({ 7: [[HASH, 0, 4, "k"]] }), { useBigInt64: true, maxDepth: 1000 })); + assert.equal(absent.chunks[0]!.id, ""); + assert.deepEqual(decodeManifest(encodeManifest(absent)).chunks[0], { chunkHash: HASH, offset: 0, sizeBytes: 4, recordKey: "k", id: "" }); + assert.equal(read(["7"]).id, "7"); + assert.throws(() => read([0]), /chunks.id/); + const packed = await packContent("x", new Uint8Array(9), { chunkSizeBytes: 4 }); + assert.deepEqual(packed.manifest.chunks.map(c => c.id), ["0", "4", "8"]); + assert.deepEqual((await packContent("x", new Uint8Array(0))).manifest.chunks.map(c => c.id), ["0"]); +}); + // Refusals that are the same refusal in the reference but would read differently if the wrong rule fired: the message // names the true cause (the sign, the nil chunk, the reserved code, the early end), never a neighbouring field. test("a refusal names its true cause", () => { @@ -433,7 +450,7 @@ test("a lone surrogate is written as U+FFFD at every string length, short and lo const withText = (value: string): CultMeshCdnArtifactManifest => ({ ...base, artifactId: value, kind: value, version: value, mimeType: value, createdAtUtc: value, - chunks: [{ ...base.chunks[0]!, recordKey: value }], + chunks: [{ ...base.chunks[0]!, chunkHash: value, recordKey: value, id: value }], tags: [value], metadata: [[value, value]], }); diff --git a/packages/cultnet-ts/test/cultnet.test.ts b/packages/cultnet-ts/test/cultnet.test.ts index a6f19800..2cc5b080 100644 --- a/packages/cultnet-ts/test/cultnet.test.ts +++ b/packages/cultnet-ts/test/cultnet.test.ts @@ -11,8 +11,10 @@ import { z } from "zod"; import { decode as decodeMsgpack, encode } from "@msgpack/msgpack"; import { CultCache, + SchemaConflictError, SingleFileMessagePackBackingStore, defineDocumentType, + schemaIdentityOf, } from "@gamecult/cultcache-ts"; import { @@ -1346,6 +1348,56 @@ test("CultNet raw replication preserves CultCache payload bytes for bit-compatib } }); +test("CultNet raw replication under a compatible schema id leaves a store that reopens and takes the next write", async () => { + const tempDir = mkdtempSync(join(tmpdir(), "cultnetts-raw-compatible-")); + + try { + const definition = defineDocumentType({ + type: "tests.raw-compatible", + schema: z.object({ name: z.string() }), + schemaId: "tests.raw-compatible.current", + schemaName: "tests.raw-compatible", + schemaVersion: "tests.raw_compatible.v1", + compatibleSchemaIds: ["tests.raw-compatible.current", "tests.raw-compatible.older"], + }); + const registry = new CultNetDocumentRegistry([ + defineCultNetDocumentBinding({ definition, payloadSchemaVersion: "tests.raw_compatible.v1" }), + ]); + const open = async (): Promise => { + const cache = CultCache.builder() + .withDocumentType(definition) + .withGenericStore(new SingleFileMessagePackBackingStore(join(tempDir, "target.msgpack"))) + .build(); + await cache.pullAllBackingStores(); + return cache; + }; + + const origin = CultCache.builder().withDocumentType(definition).build(); + await origin.put(definition, "seed", { name: "seed" }); + const payload = origin.getRequiredEnvelope(definition, "seed").payload; + + const target = await open(); + await registry.applyRawDocumentPutMessage(target, { + schemaVersion: "cultnet.document_put_raw.v0", + messageId: "raw-compatible-1", + document: { + schemaId: "tests.raw-compatible.older", + recordKey: "replicated", + storedAt: "2026-09-30T00:00:00.0000000Z", + payloadEncoding: "messagepack", + payload, + }, + } as Parameters[1]); + + const reopened = await open(); + assert.deepEqual(reopened.getRequired(definition, "replicated"), { name: "seed" }); + await reopened.put(definition, "next", { name: "next" }); + assert.deepEqual((await open()).getRequired(definition, "next"), { name: "next" }); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } +}); + test("operation envelopes preserve typed service routing and payload correlation", () => { const request = parseCultNetMessage({ schemaVersion: "cultnet.operation_request.v0", @@ -1702,162 +1754,204 @@ test("rudp sequence-neutral acknowledgements interoperate with ordered receivers assert.deepEqual(receiver.receive(request).delivered.map(frame => Buffer.from(frame.payload).toString("utf8")), ["snapshot"]); }); -test("CultNet raw replication applies compatible foreign-schema snapshots", async () => { - const tempDir = mkdtempSync(join(tmpdir(), "cultnetts-foreign-schema-raw-")); +// A replicated record whose schema id no binding declares is refused, whatever its payload parses as: the registry never +// guesses a type from a payload's shape and never relabels a record it does not understand. +test("CultNet raw replication refuses a record under a schema id no binding declares, whatever its shape", async () => { + const note = defineDocumentType({ + type: "p5.note", schemaId: "p5.note.v1", schemaName: "p5.note", schema: z.object({ body: z.string() }), + }); + // Registered first, and accepts every payload: shape routing would store the record under it. + const anything = defineDocumentType({ type: "p5.anything", schema: z.any() }); + const registry = new CultNetDocumentRegistry([ + defineCultNetDocumentBinding({ definition: anything }), + defineCultNetDocumentBinding({ definition: note }), + ]); + const origin = CultCache.builder().withDocumentType(note).build(); + await origin.put(note, "k", { body: "from a peer" }); + const snapshot = registry.createRawSnapshotResponse(origin, "p5"); + assert.equal(snapshot.documents[0]?.schemaId, "p5.note.v1"); + assert.equal(snapshot.documents[0]?.payloadEncoding, "messagepack"); - try { - const documentDefinition = defineDocumentType({ - type: "cultnet.note", - schemaId: "cultnet.note.v1", - schemaName: "cultnet.note", - schemaVersion: "cultnet.note.v1", - schema: z.object({ - schema_version: z.string(), - noteId: z.string(), - body: z.string(), + for (const schemaId of ["runtime.generated.p5.note.ui.99", "p5.note.v7", "p5.note"]) { + const target = CultCache.builder().withDocumentType(note).withDocumentType(anything).build(); + await assert.rejects( + registry.applyRawSnapshotResponse(target, { + ...snapshot, + documents: snapshot.documents.map(document => ({ ...document, schemaId })), }), - name: "noteId", - }); - const registry = new CultNetDocumentRegistry([ - defineCultNetDocumentBinding({ definition: documentDefinition }), - ]); - const originCache = CultCache.builder() - .withDocumentType(documentDefinition) - .withGenericStore(new SingleFileMessagePackBackingStore(join(tempDir, "origin.msgpack"))) - .build(); - const targetCache = CultCache.builder() - .withDocumentType(documentDefinition) - .withGenericStore(new SingleFileMessagePackBackingStore(join(tempDir, "target.msgpack"))) - .build(); - - await originCache.put(documentDefinition, "note:foreign-schema", { - schema_version: "cultnet.note.v1", - noteId: "note:foreign-schema", - body: "accepted by payload shape", - }); - const rawSnapshot = registry.createRawSnapshotResponse(originCache, "foreign-schema-raw"); - rawSnapshot.documents[0]!.schemaId = "runtime.generated.cultnet.note.ui.99"; - - await registry.applyRawSnapshotResponse(targetCache, rawSnapshot); - - assert.equal( - targetCache.getRequired(documentDefinition, "note:foreign-schema").body, - "accepted by payload shape", + (error: unknown) => error instanceof Error + && error.message === `No CultNet document binding is registered for schema "${schemaId}".`, ); - assert.equal( - targetCache.getRequiredEnvelope(documentDefinition, "note:foreign-schema").schemaId, - "cultnet.note.v1", - ); - } finally { - rmSync(tempDir, { recursive: true, force: true }); + assert.deepEqual(target.snapshot(), [], `nothing was stored for ${schemaId}`); } }); -test("CultNet document registry filters snapshots by schema aliases", async () => { - const tempDir = mkdtempSync(join(tmpdir(), "cultnetts-alias-snapshot-")); +// The registry stamps and resolves a definition's schema id exactly as CultCache does, so a record a cache wrote is read +// back as the type that wrote it on every peer. +test("CultNet registry derives a definition's schema id as the cache does, and refuses what the cache refuses", async () => { + const shape = z.object({ body: z.string() }); + const a = defineDocumentType({ type: "p1.a", schemaName: "p1.shared", schema: shape }); + const b = defineDocumentType({ type: "p1.b", schemaId: "p1.shared", schemaName: "p1.b", schema: shape }); + const fillers = [0, 1].map(index => defineCultNetDocumentBinding({ + definition: defineDocumentType({ type: `p1.filler${index}`, schema: shape }), + })); - try { - const documentDefinition = defineDocumentType({ - type: "cultnet.alias_note", - schemaId: "sha256:cultnet-alias-note", - schemaName: "cultnet.alias_note", - schemaVersion: "cultnet.alias_note.v1", - schema: z.object({ - schema_version: z.string(), - noteId: z.string(), - body: z.string(), - }), - name: "noteId", + assert.deepEqual(schemaIdentityOf(a), { + schemaId: "p1.shared", schemaName: "p1.shared", schemaVersion: "p1.shared.v1", compatibleSchemaIds: ["p1.shared"], + }); + // The cache refuses B beside A: A's records carry "p1.shared". + await assert.rejects(CultCache.builder().withDocumentType(a).build().registerDocumentType(b), SchemaConflictError); + for (const [first, second] of [[a, b], [b, a]]) { + // Two bindings stand ahead of the holder, so a check that looks only at the first holder misses it. + const registry = new CultNetDocumentRegistry([...fillers, defineCultNetDocumentBinding({ definition: first })]); + assert.throws(() => registry.register(defineCultNetDocumentBinding({ definition: second })), (error: unknown) => { + assert.ok(error instanceof SchemaConflictError, String(error)); + assert.equal(error.schemaId, "p1.shared"); + return true; }); - const registry = new CultNetDocumentRegistry([ - defineCultNetDocumentBinding({ definition: documentDefinition }), - ]); - const cache = CultCache.builder() - .withDocumentType(documentDefinition) - .withGenericStore(new SingleFileMessagePackBackingStore(join(tempDir, "alias.msgpack"))) - .build(); - - await cache.put(documentDefinition, "note:alias", { - schema_version: "cultnet.alias_note.v1", - noteId: "note:alias", - body: "filtered by alias", - }); - - const typedSnapshot = registry.createSnapshotResponse(cache, "typed-alias", { - schemaVersion: "cultnet.snapshot_request.v0", - messageId: "typed-alias", - schemaIds: ["cultnet.alias_note.v1"], - }); - const rawSnapshot = registry.createRawSnapshotResponse(cache, "raw-alias", { - schemaVersion: "cultnet.snapshot_request.v0", - messageId: "raw-alias", - schemaIds: ["cultnet.alias_note.v1"], - }); - - assert.equal(typedSnapshot.documents[0]?.schemaId, "sha256:cultnet-alias-note"); - assert.equal(typedSnapshot.documents[0]?.recordKey, "note:alias"); - assert.equal(rawSnapshot.documents[0]?.schemaId, "sha256:cultnet-alias-note"); - assert.equal(rawSnapshot.documents[0]?.recordKey, "note:alias"); - } finally { - rmSync(tempDir, { recursive: true, force: true }); + assert.equal(registry.get(second.type), undefined); } + + const registry = new CultNetDocumentRegistry([...fillers, defineCultNetDocumentBinding({ definition: a })]); + const origin = CultCache.builder().withDocumentType(a).build(); + await origin.put(a, "k", { body: "written by a" }); + const stamped = origin.getRequiredEnvelope(a, "k").schemaId; + assert.equal(stamped, "p1.shared"); + assert.equal(registry.createDocumentPutMessage(registry.get("p1.a")!, "m", "k", { body: "x" }).document.schemaId, stamped); + assert.equal(registry.createRawDocumentPutMessage(registry.get("p1.a")!, "m", "k", { body: "x" }).document.schemaId, stamped); + assert.equal(registry.getBySchemaId(stamped!)?.definition, a); + + const peer = CultCache.builder().withDocumentType(a).build(); + await registry.applyRawSnapshotResponse(peer, registry.createRawSnapshotResponse(origin, "p1")); + assert.deepEqual(peer.getRequired(a, "k"), { body: "written by a" }); + assert.equal(peer.getRequiredEnvelope(a, "k").schemaId, stamped); }); -test("CultNet document registry applies typed put and delete messages by schema alias", async () => { - const tempDir = mkdtempSync(join(tmpdir(), "cultnetts-alias-typed-")); +// An id nobody owns resolves to the one binding that lists it as compatible. Several listers and no owner resolve to +// none, and the refusal does not depend on the order they were registered in. An owner beats every lister. +test("CultNet registry refuses an unowned schema id that several bindings list, in any order", async () => { + const shape = z.object({ body: z.string() }); + const first = defineDocumentType({ type: "p2.first", schemaId: "p2.first", compatibleSchemaIds: ["p2.old"], schema: shape }); + const second = defineDocumentType({ type: "p2.second", schemaId: "p2.second", compatibleSchemaIds: ["p2.old"], schema: shape }); + const owner = defineDocumentType({ type: "p2.owner", schemaId: "p2.old", schema: shape }); + const bind = (definition: typeof first) => defineCultNetDocumentBinding({ definition }); - try { - const documentDefinition = defineDocumentType({ - type: "cultnet.typed_alias_note", - schemaId: "sha256:cultnet-typed-alias-note", - schemaName: "cultnet.typed_alias_note", - schemaVersion: "cultnet.typed_alias_note.v1", - schema: z.object({ - schema_version: z.string(), - noteId: z.string(), - body: z.string(), - }), - name: "noteId", + for (const order of [[first, second], [second, first]]) { + const registry = new CultNetDocumentRegistry(order.map(bind)); + assert.throws(() => registry.getBySchemaId("p2.old"), (error: unknown) => { + assert.ok(error instanceof SchemaConflictError, String(error)); + assert.equal(error.schemaId, "p2.old"); + assert.deepEqual(error.schemaNames, ["p2.first", "p2.second"]); + assert.equal(error.recordKey, ""); + assert.ok(error.message.includes("(p2.first, p2.second)"), error.message); + assert.ok(error.message.includes("so it resolves to none. Bind the type that owns the id"), error.message); + return true; }); - const registry = new CultNetDocumentRegistry([ - defineCultNetDocumentBinding({ definition: documentDefinition }), - ]); - const cache = CultCache.builder() - .withDocumentType(documentDefinition) - .withGenericStore(new SingleFileMessagePackBackingStore(join(tempDir, "typed-alias.msgpack"))) - .build(); + const cache = CultCache.builder().withDocumentType(first).withDocumentType(second).build(); + const origin = CultCache.builder().withDocumentType(first).build(); + await origin.put(first, "k", { body: "old" }); + const document = { ...registry.createRawSnapshotResponse(origin, "p2").documents[0]!, schemaId: "p2.old" }; + await assert.rejects( + registry.applyRawDocumentPutMessage(cache, { schemaVersion: "cultnet.document_put_raw.v0", messageId: "p2", document }), + SchemaConflictError, + ); + assert.deepEqual(cache.snapshot(), []); - await registry.applyDocumentPutMessage(cache, { + for (const withOwner of [[...order, owner], [owner, ...order]]) { + assert.equal(new CultNetDocumentRegistry(withOwner.map(bind)).getBySchemaId("p2.old")?.definition, owner); + } + } + // One lister, bound twice, is still one lister. + const single = new CultNetDocumentRegistry([bind(first), bind(first)]); + assert.equal(single.getBySchemaId("p2.old")?.definition, first); + // A snapshot filtered by an id a binding lists returns that binding's records under their own id. + const origin = CultCache.builder().withDocumentType(first).build(); + await origin.put(first, "k", { body: "listed" }); + const filter = { schemaVersion: "cultnet.snapshot_request.v0" as const, messageId: "f", schemaIds: ["p2.old"] }; + assert.deepEqual(single.createRawSnapshotResponse(origin, "f", filter).documents.map(document => document.schemaId), ["p2.first"]); + assert.deepEqual(single.createSnapshotResponse(origin, "f", filter).documents.map(document => document.schemaId), ["p2.first"]); +}); + +// A schema id is only ever an id a definition declares. A version string, a schema name, or an id that differs from a +// declared one by its version suffix names nothing. +test("CultNet registry resolves only declared schema ids, never a version, a name or an inferred version", async () => { + const note = defineDocumentType({ + type: "p3.note", + schemaId: "p3.note.v1", + schemaName: "p3.note", + schemaVersion: "p4.note.v3", + schema: z.object({ body: z.string() }), + }); + const registry = new CultNetDocumentRegistry([defineCultNetDocumentBinding({ definition: note })]); + const cache = CultCache.builder().withDocumentType(note).build(); + await cache.put(note, "k", { body: "held" }); + + assert.equal(registry.getBySchemaId("p3.note.v1")?.definition, note); + for (const undeclared of ["p3.note.v7", "p4.note.v3", "p3.note"]) { + assert.equal(registry.getBySchemaId(undeclared), undefined, undeclared); + const filter = { schemaVersion: "cultnet.snapshot_request.v0" as const, messageId: "f", schemaIds: [undeclared] }; + assert.deepEqual(registry.createSnapshotResponse(cache, "f", filter).documents, [], undeclared); + assert.deepEqual(registry.createRawSnapshotResponse(cache, "f", filter).documents, [], undeclared); + await assert.rejects(registry.applyDocumentPutMessage(cache, { schemaVersion: "cultnet.document_put.v0", - messageId: "typed-alias-put", - document: { - schemaId: "cultnet.typed_alias_note.v1", - recordKey: "note:typed-alias", - storedAt: "2026-06-27T00:00:00.000Z", - payload: { - schema_version: "cultnet.typed_alias_note.v1", - noteId: "note:typed-alias", - body: "typed alias applied", - }, - }, - }); - - assert.equal( - cache.getRequired(documentDefinition, "note:typed-alias").body, - "typed alias applied", - ); - assert.equal(registry.getBySchemaId("cultnet.typed_alias_note.v1")?.definition.type, documentDefinition.type); - - assert.equal(await registry.applyDocumentDeleteMessage(cache, { - schemaVersion: "cultnet.document_delete.v0", - messageId: "typed-alias-delete", - schemaId: "cultnet.typed_alias_note.v1", - recordKey: "note:typed-alias", - }), true); - assert.equal(cache.get(documentDefinition, "note:typed-alias"), undefined); - } finally { - rmSync(tempDir, { recursive: true, force: true }); + messageId: "put", + document: { schemaId: undeclared, recordKey: "k", storedAt: "2026-09-30T00:00:00.000Z", payload: { body: "relabelled" } }, + }), /No CultNet document binding/); + await assert.rejects(registry.applyDocumentDeleteMessage(cache, { + schemaVersion: "cultnet.document_delete.v0", messageId: "delete", schemaId: undeclared, recordKey: "k", + }), /No CultNet document binding/); + assert.deepEqual(cache.getRequired(note, "k"), { body: "held" }); } + const filter = { schemaVersion: "cultnet.snapshot_request.v0" as const, messageId: "f", schemaIds: ["p3.note.v1"] }; + assert.equal(registry.createSnapshotResponse(cache, "f", filter).documents.length, 1); + assert.equal(registry.createRawSnapshotResponse(cache, "f", filter).documents.length, 1); +}); + +test("CultNet document registry binds one type per schema and refuses a second, typed", () => { + const noteShape = z.object({ noteId: z.string() }); + const owner = defineDocumentType({ type: "tests.note", schemaId: "id.note", schemaName: "tests.note", schema: noteShape }); + const sameId = defineDocumentType({ type: "tests.note-ui", schemaId: "id.note", schemaName: "tests.note-ui", schema: noteShape }); + const sameName = defineDocumentType({ type: "tests.note-named", schemaId: "id.other", schemaName: "tests.note", schema: noteShape }); + const sameType = defineDocumentType({ type: "tests.note", schemaId: "id.third", schemaName: "tests.third", schema: noteShape }); + const lister = defineDocumentType({ + type: "tests.note-lister", schemaId: "id.lister", schemaName: "tests.note-lister", compatibleSchemaIds: ["id.note"], schema: noteShape, + }); + + for (const [claimant, schemaId, names] of [ + [sameId, "id.note", ["tests.note", "tests.note-ui"]], + [sameName, "id.other", ["tests.note", "tests.note"]], + [sameType, "id.third", ["tests.note", "tests.third"]], + ] as const) { + for (const order of [[owner, claimant], [claimant, owner]]) { + // Two bindings are registered ahead of the holder, so a conflict loop that checks only the first holder fails. + const registry = new CultNetDocumentRegistry([ + ...["tests.filler-a", "tests.filler-b"].map(type => defineCultNetDocumentBinding({ + definition: defineDocumentType({ type, schema: noteShape }), + })), + defineCultNetDocumentBinding({ definition: order[0] }), + ]); + assert.throws(() => registry.register(defineCultNetDocumentBinding({ definition: order[1] })), (error: unknown) => { + assert.ok(error instanceof SchemaConflictError, String(error)); + const ownerFirst = order[0] === owner; + assert.equal(error.schemaId, ownerFirst ? schemaId : "id.note"); + assert.deepEqual(error.schemaNames, ownerFirst ? names : [...names].reverse()); + assert.equal(error.recordKey, ""); + assert.ok(error.message.includes(`schema id "${error.schemaId}"`), error.message); + return true; + }); + // The refused binding left nothing behind: the schema still resolves to the first type. + assert.equal(registry.getBySchemaId(order[0].schemaId!)?.definition, order[0]); + assert.equal(registry.get(order[1].type)?.definition, order[1].type === order[0].type ? order[0] : undefined); + } + } + + // The same definition may be bound again, and a definition listing the id as compatible is not a second owner. + const registry = new CultNetDocumentRegistry([defineCultNetDocumentBinding({ definition: owner })]); + registry.register(defineCultNetDocumentBinding({ definition: owner, payloadSchemaVersion: "tests.note.v2" })); + registry.register(defineCultNetDocumentBinding({ definition: lister })); + assert.equal(registry.get("tests.note")?.payloadSchemaVersion, "tests.note.v2"); + assert.equal(registry.getBySchemaId("id.note")?.definition, owner); + assert.equal(registry.getBySchemaId("id.lister")?.definition, lister); }); test("CultNet interop slot compatibility defaults missing trailing fields and rejects mismatched slots", () => { diff --git a/packages/cultnet-ts/test/idunn-runtime-authority.test.ts b/packages/cultnet-ts/test/idunn-runtime-authority.test.ts index 2aff6d92..cf1e4a26 100644 --- a/packages/cultnet-ts/test/idunn-runtime-authority.test.ts +++ b/packages/cultnet-ts/test/idunn-runtime-authority.test.ts @@ -3,8 +3,9 @@ import assert from "node:assert/strict"; import { spawnSync } from "node:child_process"; import dgram from "node:dgram"; import fs from "node:fs"; +import os from "node:os"; import path from "node:path"; -import { documentPayloadBytes, documentPayloadFromBytes, type DocumentPayload } from "@gamecult/cultcache-ts"; +import { StoreUnreadableError, documentPayloadBytes, documentPayloadFromBytes, type DocumentPayload } from "@gamecult/cultcache-ts"; import { decode, encode } from "@msgpack/msgpack"; import { @@ -55,7 +56,7 @@ function expectedFacts(set: FixtureSet) { function openAuthority( context: TestContext, set: FixtureSet, - options: { machineId?: "etc" | "dbus"; lease?: string } = {}, + options: { machineId?: "etc" | "dbus"; lease?: string; bundle?: string } = {}, ): IdunnRuntimeAuthority { const facts = expectedFacts(set); const originalEnvironment = { ...process.env }; @@ -63,7 +64,7 @@ function openAuthority( for (const key of Object.keys(process.env)) if (!(key in originalEnvironment)) delete process.env[key]; Object.assign(process.env, originalEnvironment); }); - process.env.GAMECULT_IDUNN_RUNTIME_BUNDLE = path.join(FIXTURES, set); + process.env.GAMECULT_IDUNN_RUNTIME_BUNDLE = options.bundle ?? path.join(FIXTURES, set); process.env.GAMECULT_IDUNN_CANDIDATE_BIND = facts.candidate; if (options.lease) process.env.GAMECULT_IDUNN_PROCESS_WRITE_LEASE = path.join(FIXTURES, set, options.lease); else delete process.env.GAMECULT_IDUNN_PROCESS_WRITE_LEASE; @@ -85,6 +86,22 @@ function openAuthority( return loadIdunnRuntimeAuthorityFromEnvironment(facts.target, facts.healthContract); } +/** A copy of one Rust-written bundle in a scratch directory, with `change` applied to the bytes of the named file. */ +function bundleWith(context: TestContext, set: FixtureSet, file: string, change: (bytes: Buffer) => Buffer | undefined): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "idunn-authority-")); + context.after(() => fs.rmSync(dir, { recursive: true, force: true })); + for (const name of ["expected.cc", "activation.cc"]) { + const bytes = fixtureFile(set, name); + const changed = name === file ? change(bytes) : bytes; + if (changed !== undefined) fs.writeFileSync(path.join(dir, name), changed); + } + return dir; +} + +function withHeader(header: string): (bytes: Buffer) => Buffer { + return (bytes) => Buffer.from(encode([header, ...(decode(bytes) as unknown[]).slice(1)])); +} + function signerFor(context: TestContext, set: FixtureSet, capacityDelta = 0, lease?: string) { const authority = openAuthority(context, set, { lease }); const capabilities = expectedFacts(set).capabilities.map((capability) => ({ ...capability, capacity: capability.capacity + capacityDelta })); @@ -134,6 +151,70 @@ test("rejects a credential file holding more than one envelope", (context) => { ); }); +// The authority files are read by cultcache-ts's one store reader, so a store it reads is read here and a file it +// refuses is refused here with the same typed error and the same text. +for (const file of ["expected.cc", "activation.cc"]) { + test(`${file}: a store marked for element ids opens, as the one reader reads it`, (context) => { + const bundle = bundleWith(context, "web", file, withHeader("cultcache.store.v3")); + assert.equal(openAuthority(context, "web", { bundle }).expected.target, expectedFacts("web").target); + }); + + test(`${file}: a zero-byte file is refused as unreadable, not read as an empty store or a missing one`, (context) => { + const bundle = bundleWith(context, "web", file, () => Buffer.alloc(0)); + assert.throws(() => openAuthority(context, "web", { bundle }), StoreUnreadableError); + }); + + test(`${file}: a file that is not there is refused`, (context) => { + const bundle = bundleWith(context, "web", file, () => undefined); + assert.throws(() => openAuthority(context, "web", { bundle }), /does not exist/); + }); + + test(`${file}: a refused header is never echoed, whether or not it starts at the store prefix`, (context) => { + for (const header of ["cultcache.store.SECRET", "cultcache.store.SECRETcultcache.store.v9", "SECRETcultcache.store.v9", "cultcache.store.v9"]) { + const bundle = bundleWith(context, "web", file, withHeader(header)); + assert.throws( + () => openAuthority(context, "web", { bundle }), + (error: Error) => error instanceof StoreUnreadableError && (header === "cultcache.store.v9" ? error.message.includes(header) : !error.message.includes("SECRET")), + header, + ); + } + }); +} + +// What an authority file may hold once the one reader has accepted it: exactly one record, under the schema +// id and the type the loader asked for. Each case edits the Rust-written store and changes one thing. +function withStore(change: (store: unknown[][]) => void): (bytes: Buffer) => Buffer { + return (bytes) => { + const store = decode(bytes) as unknown[][]; + change(store); + return Buffer.from(encode(store)); + }; +} + +test("an authority file holding two records is refused", (context) => { + const bundle = bundleWith(context, "web", "expected.cc", withStore((store) => { + const second = [...(store[2]![0] as unknown[])]; + second[0] = "second"; + store[2]!.push(second); + })); + assert.throws(() => openAuthority(context, "web", { bundle }), /must contain exactly one record/); +}); + +test("an authority file whose one record is under another schema id is refused", (context) => { + const bundle = bundleWith(context, "web", "expected.cc", () => fixtureFile("web", "activation.cc")); + assert.throws( + () => openAuthority(context, "web", { bundle }), + (error: Error) => /must contain exactly one .+ record/.test(error.message) && !error.message.includes("unexpected CultCache type"), + ); +}); + +test("an authority record whose catalog names another type is refused, naming that type", (context) => { + const bundle = bundleWith(context, "web", "expected.cc", withStore((store) => { + (store[1]![0] as unknown[])[1] = "idunn.other_type"; + })); + assert.throws(() => openAuthority(context, "web", { bundle }), /unexpected CultCache type idunn.other_type/); +}); + function presenceSlot(document: { payload: DocumentPayload }, slot: number): unknown { return (decode(documentPayloadBytes(document.payload)) as unknown[])[slot]; } diff --git a/packages/cultnet-ts/test/interop/cultnet-interop.test.ts b/packages/cultnet-ts/test/interop/cultnet-interop.test.ts index 598d6fe6..d0e732ce 100644 --- a/packages/cultnet-ts/test/interop/cultnet-interop.test.ts +++ b/packages/cultnet-ts/test/interop/cultnet-interop.test.ts @@ -1286,9 +1286,9 @@ test("CultNet TS/Rust/C#/Python peers discover each other and exchange raw state schemaVersion: INTEROP_SCHEMA_VERSION, schemaId: interopNoteSchemaId, }, - [{ role: "payload", schemaId: interopNoteSchemaId, schemaVersion: INTEROP_SCHEMA_VERSION }], - [{ role: "log", uri: "cultcache://bundle:ts-python-witness/log", mediaType: "text/plain" }], - [{ stage: "roundtrip", startedAt: "2026-06-13T00:00:03Z", completedAt: "2026-06-13T00:00:04Z", latencyMs: 1 }], + [{ id: "c0e1a2b3c4d5", role: "payload", schemaId: interopNoteSchemaId, schemaVersion: INTEROP_SCHEMA_VERSION }], + [{ id: "a0e1a2b3c4d5", role: "log", uri: "cultcache://bundle:ts-python-witness/log", mediaType: "text/plain" }], + [{ id: "e0e1a2b3c4d5", stage: "roundtrip", startedAt: "2026-06-13T00:00:03Z", completedAt: "2026-06-13T00:00:04Z", latencyMs: 1 }], { pipelineId: "interop", runId: "ts-python-witness", runtimeId: "ts-witness", agentId: "ts-agent" }, ]), sourceRuntimeId: "ts-witness", @@ -1306,6 +1306,7 @@ test("CultNet TS/Rust/C#/Python peers discover each other and exchange raw state assert.equal(witnessSlots[1], "interop-proof"); assert.equal(witnessSlots[3].subjectId, "note:python-peer"); assert.equal(witnessSlots[7].runtimeId, "ts-witness"); + assert.deepEqual([witnessSlots[4][0].id, witnessSlots[5][0].id, witnessSlots[6][0].id], ["c0e1a2b3c4d5", "a0e1a2b3c4d5", "e0e1a2b3c4d5"], "element ids cross TS to Python and back"); const expectedPeers = ["csharp-peer", "python-peer", "rust-peer", "ts-peer"]; diff --git a/scripts/test-declared-dependency-ranges.mjs b/scripts/test-declared-dependency-ranges.mjs new file mode 100644 index 00000000..b528fb75 --- /dev/null +++ b/scripts/test-declared-dependency-ranges.mjs @@ -0,0 +1,155 @@ +// Observes the dependency ranges the TypeScript workspace packages declare, not the workspace symlinks that satisfy them. +// +// Every workspace test resolves a sibling package through node_modules/ -> packages/, so a package that starts +// importing a new export from a sibling passes whatever range it declares. A consumer outside the workspace installs the +// newest published version the range admits. This check builds every tagged release (`-v`) that a declared +// range admits and requires each runtime value the dependent imports from that package to be an export of it. +// +// node scripts/test-declared-dependency-ranges.mjs +// +// It needs the workspace installed (typescript at the repo root) and git with the release tags. A package whose range +// admits no tagged release other than the workspace's own passes, because nothing a consumer could install lacks the export. +// Namespace imports, default imports, type-only imports and subpath imports are not checked. + +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, symlinkSync } from "node:fs"; +import { createRequire } from "node:module"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const rootRequire = createRequire(join(repoRoot, "package.json")); +const ts = rootRequire("typescript"); +const semver = createRequire(join(npmRoot(), "package.json"))("semver"); + +// npm ships semver, so the check needs no dependency of its own. npm's root is beside the node binary on Windows and under lib/ on Unix. +function npmRoot() { + const candidates = [ + ...(process.env.npm_execpath ? [dirname(dirname(process.env.npm_execpath))] : []), + join(dirname(process.execPath), "node_modules", "npm"), + join(dirname(process.execPath), "..", "lib", "node_modules", "npm"), + ]; + const root = candidates.find((candidate) => existsSync(join(candidate, "package.json"))); + assert.ok(root, `npm's package root was not found in ${candidates.join(", ")}`); + return root; +} + +function readJson(path) { + return JSON.parse(readFileSync(path, "utf8")); +} + +function git(...args) { + return execFileSync("git", args, { cwd: repoRoot, encoding: "utf8" }); +} + +// The runtime names `sourceText` imports from, or re-exports from, exactly `packageName`. +function importedNames(sourceText, packageName) { + const file = ts.createSourceFile("source.ts", sourceText, ts.ScriptTarget.Latest, true); + const names = new Set(); + for (const statement of file.statements) { + const from = statement.moduleSpecifier; + if (!from || !ts.isStringLiteral(from) || from.text !== packageName) { + continue; + } + + let clause; + if (ts.isImportDeclaration(statement) && statement.importClause && !statement.importClause.isTypeOnly) { + clause = statement.importClause.namedBindings; + } else if (ts.isExportDeclaration(statement) && !statement.isTypeOnly) { + clause = statement.exportClause; + } + + if (clause && (ts.isNamedImports(clause) || ts.isNamedExports(clause))) { + for (const element of clause.elements) { + if (!element.isTypeOnly) { + names.add((element.propertyName ?? element.name).text); + } + } + } + } + + return [...names].sort(); +} + +function sourceFiles(directory) { + return readdirSync(directory, { withFileTypes: true }).flatMap((entry) => { + const path = join(directory, entry.name); + if (entry.isDirectory()) { + return sourceFiles(path); + } + + return entry.name.endsWith(".ts") && !entry.name.endsWith(".d.ts") ? [path] : []; + }); +} + +// Builds the package as the tag released it, in a scratch directory that borrows the workspace's node_modules. +function buildRelease(directory, tag, scratch) { + const archive = join(scratch, `${tag}.tar`); + git("archive", "--format=tar", "-o", archive, tag, `packages/${directory}`); + const extracted = join(scratch, tag); + mkdirSync(extracted); + execFileSync("tar", ["-xf", archive, "-C", extracted]); + symlinkSync(join(repoRoot, "node_modules"), join(extracted, "node_modules"), "junction"); + const packageRoot = join(extracted, "packages", directory); + execFileSync(process.execPath, [rootRequire.resolve("typescript/bin/tsc"), "-p", "tsconfig.json"], { cwd: packageRoot, stdio: "inherit" }); + return rootRequire(join(packageRoot, "dist", "index.js")); +} + +const workspaces = readJson(join(repoRoot, "package.json")).workspaces.map((directory) => { + const manifest = readJson(join(repoRoot, directory, "package.json")); + return { directory: directory.replace(/^packages\//u, ""), root: join(repoRoot, directory), manifest }; +}); +const byName = new Map(workspaces.map((workspace) => [workspace.manifest.name, workspace])); +const tags = git("tag", "-l").split(/\r?\n/u).filter(Boolean); +assert.ok(tags.length > 0, "this clone has no release tags; fetch them (git fetch --tags) so the admitted releases can be built"); +const scratch = mkdtempSync(join(tmpdir(), "cultlib-declared-ranges-")); +const built = new Map(); +let checked = 0; + +try { + for (const dependent of workspaces) { + for (const [name, range] of Object.entries(dependent.manifest.dependencies ?? {})) { + const dependency = byName.get(name); + if (!dependency) { + continue; + } + + assert.ok( + semver.satisfies(dependency.manifest.version, range), + `${dependent.manifest.name} declares ${name} ${range}, which excludes the workspace's own ${dependency.manifest.version}`, + ); + const src = join(dependent.root, "src"); + const imported = existsSync(src) + ? [...new Set(sourceFiles(src).flatMap((file) => importedNames(readFileSync(file, "utf8"), name)))].sort() + : []; + const prefix = `${dependency.directory}-v`; + const admitted = tags.filter((tag) => { + const version = tag.startsWith(prefix) ? tag.slice(prefix.length) : undefined; + return version !== undefined && semver.valid(version) !== null && semver.satisfies(version, range) && version !== dependency.manifest.version; + }); + for (const tag of admitted) { + if (!built.has(tag)) { + built.set(tag, buildRelease(dependency.directory, tag, scratch)); + } + + const exports = built.get(tag); + const missing = imported.filter((symbol) => !(symbol in exports)); + assert.deepEqual( + missing, + [], + `${dependent.manifest.name} declares ${name} ${range}, which admits ${tag}, and ${tag} does not export ${missing.join(", ")}`, + ); + } + + checked += 1; + console.log(`${dependent.manifest.name} -> ${name} ${range}: ${imported.length} imported names, ${admitted.length} admitted tagged releases checked`); + } + } + + assert.ok(checked > 0, "no workspace package declares a dependency on another workspace package"); + console.log(`Declared dependency ranges: ${checked} checked`); +} finally { + rmSync(scratch, { recursive: true, force: true }); +} diff --git a/src/GameCult.Caching.MessagePack/CultCacheMessagePack.cs b/src/GameCult.Caching.MessagePack/CultCacheMessagePack.cs index 4fa55076..8d0c7c96 100644 --- a/src/GameCult.Caching.MessagePack/CultCacheMessagePack.cs +++ b/src/GameCult.Caching.MessagePack/CultCacheMessagePack.cs @@ -23,10 +23,10 @@ namespace GameCult.Caching.MessagePack // options, resolvers included, so dictionary keys compare by the bytes the store writes; a document type // deserializes as a registered document, a member type as itself. public static CultCodec CreateCodec(CultDocumentRegistry registry) => new CultCodec( - (value, type, document) => global::MessagePack.MessagePackSerializer.Serialize(type, value, CultDocumentMessagePackSerialization.OptionsFor(document.Assembly)), + (value, type, document) => global::MessagePack.MessagePackSerializer.Serialize(type, value, CultDocumentMessagePackSerialization.OptionsForDocument(document)), (type, document, bytes) => type == document ? CultDocumentMessagePackSerialization.DeserializeUntyped(type, bytes, registry) - : global::MessagePack.MessagePackSerializer.Deserialize(type, bytes, CultDocumentMessagePackSerialization.OptionsFor(document.Assembly)), + : global::MessagePack.MessagePackSerializer.Deserialize(type, bytes, CultDocumentMessagePackSerialization.OptionsForDocument(document)), CultDocumentMessagePackSerialization.OverlaySlots); // The inspection model over that codec. diff --git a/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs b/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs index 9db26a97..55fd7584 100644 --- a/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs +++ b/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs @@ -1,4 +1,5 @@ using System; +using System.IO; using System.Buffers; using System.Collections.Concurrent; using System.Collections.Generic; @@ -37,11 +38,23 @@ public static class CultDocumentMessagePackSerialization private const int SchemaCatalogMemberFieldCount = 8; private const int StoreSnapshotFieldCount = 3; - public static readonly MessagePackSerializerOptions Options = Compose(Array.Empty()); + private static readonly (MessagePackSerializerOptions Options, IFormatterResolver[] Ahead) Plain = Compose(Array.Empty()); - private static readonly ConcurrentDictionary AssemblyOptions = new(); + public static readonly MessagePackSerializerOptions Options = Plain.Options; - public static MessagePackSerializerOptions OptionsFor(Assembly documentAssembly) + // Per assembly: the options a document type's bytes are written with, and the resolvers composed ahead of + // StandardResolver, the ones that can claim a type before its [Key] layout does. + private static readonly ConcurrentDictionary AssemblyOptions = new(); + + // Per document type: why its element ids cannot reach disk under the composed resolvers, or null. Types and resolvers + // are fixed for the process, so this is a cache of a pure answer. + private static readonly ConcurrentDictionary DocumentProblems = new(); + + private static readonly MethodInfo GetFormatterMethod = typeof(IFormatterResolver).GetMethod(nameof(IFormatterResolver.GetFormatter))!; + + public static MessagePackSerializerOptions OptionsFor(Assembly documentAssembly) => ComposedFor(documentAssembly).Options; + + private static (MessagePackSerializerOptions Options, IFormatterResolver[] Ahead) ComposedFor(Assembly documentAssembly) { if (documentAssembly == null) throw new ArgumentNullException(nameof(documentAssembly)); return AssemblyOptions.GetOrAdd(documentAssembly, static assembly => @@ -49,19 +62,118 @@ public static class CultDocumentMessagePackSerialization var resolvers = assembly.GetCustomAttributes() .Select(attribute => CreateResolver(attribute.ResolverType)) .ToArray(); - return resolvers.Length == 0 ? Options : Compose(resolvers); + return resolvers.Length == 0 ? Plain : Compose(resolvers); }); } - private static MessagePackSerializerOptions Compose(IFormatterResolver[] consumerResolvers) + // The options a document's bytes are written and read with. A document's list elements carry their ids by their [Key] + // layout, so a resolver ahead of StandardResolver that formats a type on the way to those lists would write them + // without the id: refused here, once per document type, on every encode and decode path. + internal static MessagePackSerializerOptions OptionsForDocument(Type documentType) + { + if (documentType == null) throw new ArgumentNullException(nameof(documentType)); + var (options, ahead) = ComposedFor(documentType.Assembly); + var problem = DocumentProblems.GetOrAdd(documentType, type => ResolverClaim(type, ahead)); + return problem == null ? options : throw new InvalidOperationException(problem); + } + + private static string? ResolverClaim(Type documentType, IFormatterResolver[] ahead) + { + foreach (var type in CultElementIds.Inspect(documentType).IdPath) + { + var generic = GetFormatterMethod.MakeGenericMethod(type); + // A resolver that hands back MessagePack's own formatter for the type writes its [Key] layout: that is no claim, and the + // composite stops there, so no later resolver is asked. StandardResolverAllowPrivate's formatter writes the same layout + // plus non-public [Key] members; those extra members are whole values the walk does not own, so it claims nothing + // either. + object? standard; + try + { + standard = generic.Invoke(StandardResolver.Instance, null); + } + catch (Exception exception) + { + return CannotBuild(documentType, type, (exception.InnerException ?? exception).Message); + } + + if (standard == null) + return CannotBuild(documentType, type, "StandardResolver has no formatter for it"); + + if (WrittenAsCollection(type, generic, standard)) + return $"Cult document {documentType.Name}: MessagePack writes {type.Name} as a collection, but the element-id walk reads it by its [Key] layout, so element ids would be lost or invented. A [MessagePackObject] type that is also a collection (ICollection, IList, IDictionary) is written as a collection; make it one or the other. Or declare [MessagePackFormatter] on the type or member, which makes it one whole value with no element ids."; + + var allowPrivateAsked = false; + object? allowPrivate = null; + foreach (var resolver in ahead) + { + object? formatter; + try + { + formatter = generic.Invoke(resolver, null); + } + catch (Exception exception) + { + return $"Cult document {documentType.Name}: formatter resolver {resolver.GetType().FullName} failed to build a formatter for {type.Name}, which is on the path to a list whose elements carry ids ({(exception.InnerException ?? exception).Message}); a resolver may not throw for such a type."; + } + + if (formatter == null) + continue; + if (ReferenceEquals(formatter, standard)) + break; + if (!allowPrivateAsked) + { + allowPrivateAsked = true; + try + { + allowPrivate = generic.Invoke(StandardResolverAllowPrivate.Instance, null); + } + catch (Exception) + { + allowPrivate = null; + } + } + + if (allowPrivate != null && ReferenceEquals(formatter, allowPrivate)) + break; + return $"Cult document {documentType.Name}: formatter resolver {resolver.GetType().FullName} formats {type.Name}, which is on the path to a list whose elements carry ids, so its element ids would never reach disk. A resolver may not claim such a type; write it by its [Key] layout, keep it out of the document's lists, or declare [MessagePackFormatter] on the type or member, which makes it one whole value with no element ids."; + } + } + + return null; + } + + private static string CannotBuild(Type documentType, Type type, string reason) => + $"Cult document {documentType.Name}: MessagePack cannot build a formatter for {type.Name}, which is on the path to a list whose elements carry ids ({reason})."; + + // The door verifies the walk and never decides ownership. It refuses a proven mismatch: a type the walk reads by key + // (an owner) whose StandardResolver formatter is the very instance the generic resolver's singleton hands out (a keyed + // ICollection or dictionary). Union types walk as owners and the immutable and builtin resolvers claim only framework + // types, so neither can contradict. Any other formatter, a source-generated one among them, passes. + private static bool WrittenAsCollection(Type type, MethodInfo generic, object? standard) => + CultElementIds.WalksAsOwner(type) && ReferenceEquals(Formatter(DynamicGenericResolver.Instance, generic), standard); + + private static object? Formatter(IFormatterResolver resolver, MethodInfo generic) + { + try + { + return generic.Invoke(resolver, null); + } + catch (Exception) + { + return null; + } + } + + private static (MessagePackSerializerOptions Options, IFormatterResolver[] Ahead) Compose(IFormatterResolver[] consumerResolvers) { // The guard goes first and wraps whatever formatter the rest of the composite would hand out, so a consumer // resolver still encodes an arm while the key check runs. - var inner = CompositeResolver.Create( - consumerResolvers.Append(CultDocumentResolver.Instance).Append(CultMathResolver.Instance).Append(CultMathExprResolver.Instance).Append(StandardResolver.Instance).ToArray()); - return MessagePackSerializerOptions.Standard + var ahead = consumerResolvers.Append(CultDocumentResolver.Instance).Append(CultMathResolver.Instance).Append(CultMathExprResolver.Instance).ToArray(); + var inner = CompositeResolver.Create(ahead.Append(StandardResolver.Instance).ToArray()); + var options = MessagePackSerializerOptions.Standard .WithResolver(CompositeResolver.Create(new CultUnionGuard(inner), inner)) .WithSecurity(CultMessagePackSecurity.Instance); + return (options, ahead); } private static IFormatterResolver CreateResolver(Type resolverType) @@ -83,7 +195,7 @@ public static class CultDocumentMessagePackSerialization { if (registry == null) throw new ArgumentNullException(nameof(registry)); if (value != null) registry.GetRequired(type); - var payload = MessagePackSerializer.Serialize(type, value, OptionsFor(type.Assembly)); + var payload = MessagePackSerializer.Serialize(type, value, OptionsForDocument(type)); CultPayloadDepth.Check(payload, type, PayloadDirection.Write); return payload; } @@ -98,7 +210,7 @@ public static class CultDocumentMessagePackSerialization if (registry == null) throw new ArgumentNullException(nameof(registry)); registry.GetRequired(type); CultPayloadDepth.Check(payload, type, PayloadDirection.Read); - return MessagePackSerializer.Deserialize(type, payload, OptionsFor(type.Assembly)) + return MessagePackSerializer.Deserialize(type, payload, OptionsForDocument(type)) ?? throw new InvalidOperationException($"MessagePack returned null for Cult document type {type.FullName}."); } @@ -133,7 +245,7 @@ public static class CultDocumentMessagePackSerialization elements[slot] = value; } - var buffer = new ArrayBufferWriter(); + var buffer = new global::System.Buffers.ArrayBufferWriter(); var writer = new MessagePackWriter(buffer); writer.WriteArrayHeader(elements.Count); foreach (var element in elements) @@ -160,22 +272,35 @@ public static class CultDocumentMessagePackSerialization return ReadPersistedRecord(ref reader); } - public static byte[] SerializeSnapshot(CultPersistedStoreSnapshot snapshot) + /// + /// Encodes the snapshot: every catalog entry and record from its fields, so what a caller edited is what is written. + /// + public static byte[] SerializeSnapshot(CultPersistedStoreSnapshot snapshot) => + SerializeStore( + snapshot.FormatVersion, + snapshot.SchemaCatalog.Select(SerializeCatalogEntry).ToArray(), + snapshot.Records.Select(SerializePersistedRecord).ToArray()); + + // A store file from its parts, each already encoded: what a store carries forward is the bytes its file held, what it writes is + // an encoding of the object. + internal static byte[] SerializeStore(string formatVersion, IReadOnlyList catalogEntries, IReadOnlyList records) { + if (formatVersion == null) + throw new ArgumentException("A store is written under a format version; the snapshot carries none.", nameof(formatVersion)); var buffer = new global::System.Buffers.ArrayBufferWriter(); var writer = new MessagePackWriter(buffer); writer.WriteArrayHeader(StoreSnapshotFieldCount); - writer.Write(snapshot.FormatVersion); - writer.WriteArrayHeader(snapshot.SchemaCatalog.Length); - foreach (var entry in snapshot.SchemaCatalog) + writer.Write(formatVersion); + writer.WriteArrayHeader(catalogEntries.Count); + foreach (var entry in catalogEntries) { - WriteSchemaCatalogEntry(ref writer, entry); + writer.WriteRaw(new ReadOnlySpan(entry)); } - writer.WriteArrayHeader(snapshot.Records.Length); - foreach (var record in snapshot.Records) + writer.WriteArrayHeader(records.Count); + foreach (var record in records) { - WritePersistedRecord(ref writer, record); + writer.WriteRaw(new ReadOnlySpan(record)); } writer.Flush(); @@ -183,57 +308,136 @@ public static class CultDocumentMessagePackSerialization } /// - /// Parses a store snapshot whose format stamp is a string, of any version. It refuses a nil or binary stamp - /// (1.0.61 read a nil stamp as v1). A store reading its own file checks the stamp against the formats it reads, - /// before the body is parsed. + /// Reads a store file: exactly one complete MessagePack array of three slots (header, schema catalog, records), or an empty + /// array, which is an empty store. A truncated file, bytes after the array, a value that is not an array, a first slot that + /// is not a header string, a store with fewer or more slots, a record whose schema the catalog does not publish, or a body + /// that does not decode are refused with + /// , so a reader never takes part of a file for the whole and never skips a slot + /// it does not understand. The rewriting paths use this same reader as their verdict on whether a file may be replaced. /// - public static CultPersistedStoreSnapshot DeserializeSnapshot(byte[] payload) => - DeserializeSnapshot(payload, _ => { }); + public static CultPersistedStoreSnapshot DeserializeSnapshot(byte[] payload) => ReadStore(payload).Snapshot; - - // The stamp decides readability and the body never does: requireReadable runs on slot 0 before anything else is parsed. - internal static CultPersistedStoreSnapshot DeserializeSnapshot(byte[] payload, Action requireReadable) + // The reader behind DeserializeSnapshot, and beside the snapshot the exact bytes the file held for each catalog entry and each + // record (parallel to the snapshot's arrays). The snapshot's objects hold no bytes: a store that carries entries forward keeps + // these slices itself. The stamp decides readability and the body never does: requireReadable, when a store passes its rule, + // runs on slot 0 before the catalog or any record is parsed; the public parser passes none and reads any stamp. + internal static (CultPersistedStoreSnapshot Snapshot, byte[][] CatalogBytes, byte[][] RecordBytes) ReadStore( + byte[] payload, + Action? requireReadable = null) { - var reader = new MessagePackReader(payload); - var fieldCount = reader.ReadArrayHeader(); - var snapshot = new CultPersistedStoreSnapshot(); - - if (fieldCount > 0) + try { - // A stamp encoded as binary is not one this runtime wrote or reads; it is refused here, not parsed as damage. - if (reader.NextMessagePackType == MessagePackType.Binary) - throw new NotSupportedException("Store snapshot declares its format version as binary; this runtime reads " + CultPersistedStoreSnapshot.FormatV1 + " and " + CultPersistedStoreSnapshot.FormatV2 + " as strings."); - snapshot.FormatVersion = reader.ReadString() - ?? throw new NotSupportedException("Store snapshot declares no format version; this runtime reads " + CultPersistedStoreSnapshot.FormatV1 + " and " + CultPersistedStoreSnapshot.FormatV2 + "."); - requireReadable(snapshot.FormatVersion); - } + // The stamp is judged from the leading bytes alone, so a body that will not parse can never make a newer store look + // like damage. + JudgeStamp(payload, requireReadable); + RequireOneArray(payload); + var reader = new MessagePackReader(payload); + var fieldCount = reader.ReadArrayHeader(); + if (fieldCount != 0 && fieldCount != StoreSnapshotFieldCount) + { + throw new CultStoreUnreadableException( + $"The store has {fieldCount} top-level slots; this runtime reads {StoreSnapshotFieldCount} (header, schema catalog, records) or none."); + } + + var snapshot = new CultPersistedStoreSnapshot(); + if (fieldCount == 0) + return (snapshot, Array.Empty(), Array.Empty()); + + snapshot.FormatVersion = reader.ReadString()!; - if (fieldCount > 1) - { var catalogCount = reader.ReadArrayHeader(); snapshot.SchemaCatalog = new CultSchemaCatalogEntry[catalogCount]; + var catalogBytes = new byte[catalogCount][]; for (var index = 0; index < catalogCount; index++) { + var start = reader.Position; snapshot.SchemaCatalog[index] = ReadSchemaCatalogEntry(ref reader); + catalogBytes[index] = reader.Sequence.Slice(start, reader.Position).ToArray(); } - } - if (fieldCount > 2) - { var recordCount = reader.ReadArrayHeader(); snapshot.Records = new CultPersistedRecord[recordCount]; + var recordBytes = new byte[recordCount][]; for (var index = 0; index < recordCount; index++) { + var start = reader.Position; snapshot.Records[index] = ReadPersistedRecord(ref reader); + recordBytes[index] = reader.Sequence.Slice(start, reader.Position).ToArray(); } - } - for (var index = StoreSnapshotFieldCount; index < fieldCount; index++) + RequirePublishedSchemas(snapshot); + return (snapshot, catalogBytes, recordBytes); + } + catch (CultStoreUnreadableException) { - reader.Skip(); + throw; + } + catch (Exception ex) when (ex is not OutOfMemoryException) + { + throw new CultStoreUnreadableException(ex.Message, innerException: ex); + } + } + + // A record whose schema the store's catalog does not publish cannot be read by any runtime that has only the file. + private static void RequirePublishedSchemas(CultPersistedStoreSnapshot snapshot) + { + var published = new HashSet(StringComparer.Ordinal); + foreach (var entry in snapshot.SchemaCatalog) + { + published.Add(entry.SchemaId); + foreach (var compatible in entry.CompatibleSchemaIds) + published.Add(compatible); } - return snapshot; + var unpublished = snapshot.Records.FirstOrDefault(record => !published.Contains(record.SchemaId)); + if (unpublished != null) + { + throw new CultStoreUnreadableException( + $"Record '{unpublished.Key}' names schema '{unpublished.SchemaId}', which the store's schema catalog does not publish."); + } + } + + // Slot 0 of a non-empty store, read before anything after it: a binary or nil stamp is refused as such, and a string stamp is + // judged by the caller's rule when it passes one. Bytes that do not open with an array and a string are left to the full read. + private static void JudgeStamp(byte[] payload, Action? requireReadable) + { + string? stamp; + try + { + var reader = new MessagePackReader(payload); + if (reader.NextMessagePackType != MessagePackType.Array || reader.ReadArrayHeader() == 0) + return; + if (reader.NextMessagePackType == MessagePackType.Binary) + throw new CultStoreUnreadableException("Store snapshot declares its format version as binary; this runtime reads " + CultPersistedStoreSnapshot.FormatV1 + ", " + CultPersistedStoreSnapshot.FormatV2 + " and " + CultPersistedStoreSnapshot.FormatV3 + " as strings."); + if (reader.TryReadNil()) + throw new CultStoreUnreadableException("Store snapshot declares no format version; this runtime reads " + CultPersistedStoreSnapshot.FormatV1 + ", " + CultPersistedStoreSnapshot.FormatV2 + " and " + CultPersistedStoreSnapshot.FormatV3 + "."); + stamp = reader.NextMessagePackType == MessagePackType.String ? reader.ReadString() : null; + } + catch (Exception ex) when (ex is EndOfStreamException or MessagePackSerializationException or InvalidOperationException) + { + return; + } + + if (stamp != null) + requireReadable?.Invoke(stamp); + } + + private static void RequireOneArray(byte[] bytes) + { + var whole = new MessagePackReader(bytes); + try + { + if (whole.NextMessagePackType != MessagePackType.Array) + throw new CultStoreUnreadableException($"The store is a MessagePack {whole.NextMessagePackType}, not an array."); + whole.Skip(); + } + catch (Exception ex) when (ex is EndOfStreamException or MessagePackSerializationException or InvalidOperationException) + { + throw new CultStoreUnreadableException("The store is not one complete MessagePack array.", innerException: ex); + } + + if (!whole.End) + throw new CultStoreUnreadableException("The store has bytes after its MessagePack array."); } /// Refuses a single-file snapshot this runtime cannot read, naming the format or record it found. @@ -242,37 +446,49 @@ public static class CultDocumentMessagePackSerialization var version = snapshot.FormatVersion; RequireReadableSingleFileStamp(version); - // Only a v2 store may hold a variant: a v1 header over a variant record is a store no reader could trust. + // Only a v2 or v3 store may hold a variant: a v1 header over a variant record is a store no reader could trust. if (string.Equals(version, CultPersistedStoreSnapshot.FormatV1, StringComparison.Ordinal)) { var variant = snapshot.Records.FirstOrDefault(record => record.Variant != null); if (variant != null) { - throw new NotSupportedException( - $"Record '{variant.Key}' (schema '{variant.SchemaId}') is a variant but the store declares {version}; variants need {CultPersistedStoreSnapshot.FormatV2}."); + throw new CultStoreUnreadableException( + $"Record '{variant.Key}' (schema '{variant.SchemaId}') is a variant but the store declares {version}; variants need {CultPersistedStoreSnapshot.FormatV2} or {CultPersistedStoreSnapshot.FormatV3}."); } } } + // The one verdict on a single-file store's stamp, judged on slot 0 before the body is parsed: this runtime reads v1, v2 and v3. internal static void RequireReadableSingleFileStamp(string version) { if (!string.Equals(version, CultPersistedStoreSnapshot.FormatV1, StringComparison.Ordinal) && - !string.Equals(version, CultPersistedStoreSnapshot.FormatV2, StringComparison.Ordinal)) + !string.Equals(version, CultPersistedStoreSnapshot.FormatV2, StringComparison.Ordinal) && + !string.Equals(version, CultPersistedStoreSnapshot.FormatV3, StringComparison.Ordinal)) { - throw new NotSupportedException( - $"Store format {DescribeHeader(version)} is not readable; this runtime reads {CultPersistedStoreSnapshot.FormatV1} and {CultPersistedStoreSnapshot.FormatV2}."); + throw new CultStoreUnreadableException( + $"Store format {DescribeHeader(version)} is not readable; this runtime reads {CultPersistedStoreSnapshot.FormatV1}, {CultPersistedStoreSnapshot.FormatV2} and {CultPersistedStoreSnapshot.FormatV3}."); } } // A store header as a refusal may show it: echoed in the shape cultcache.store.v, and otherwise described // only by its length, since the bytes are the store's. - private static string DescribeHeader(string header) + internal static string DescribeHeader(string header) { const string prefix = "cultcache.store.v"; - var version = header.StartsWith(prefix, StringComparison.Ordinal) ? header.Substring(prefix.Length) : ""; - return version.Length > 0 && version.All(character => character >= '0' && character <= '9') - ? header - : $"an unrecognised cultcache.store.* header of {System.Text.Encoding.UTF8.GetByteCount(header)} bytes"; + if (header.StartsWith(prefix, StringComparison.Ordinal)) + { + var end = prefix.Length; + while (end < header.Length && header[end] >= '0' && header[end] <= '9') + end++; + var digits = end - prefix.Length; + if (digits is > 0 and <= 6 && (end == header.Length || header[end] == '.')) + { + var named = header.Substring(0, end); + return end == header.Length ? named : $"{named} with a suffix of {System.Text.Encoding.UTF8.GetByteCount(header) - end} bytes"; + } + } + + return $"an unrecognised cultcache.store.* header of {System.Text.Encoding.UTF8.GetByteCount(header)} bytes"; } private static void WritePersistedRecord(ref MessagePackWriter writer, CultPersistedRecord record) @@ -403,6 +619,15 @@ public static class CultDocumentMessagePackSerialization return record; } + internal static byte[] SerializeCatalogEntry(CultSchemaCatalogEntry entry) + { + var buffer = new global::System.Buffers.ArrayBufferWriter(); + var writer = new MessagePackWriter(buffer); + WriteSchemaCatalogEntry(ref writer, entry); + writer.Flush(); + return buffer.WrittenSpan.ToArray(); + } + private static void WriteSchemaCatalogEntry(ref MessagePackWriter writer, CultSchemaCatalogEntry entry) { writer.WriteArrayHeader(SchemaCatalogEntryFieldCount); @@ -555,21 +780,31 @@ public class SingleFileMessagePackBackingStore : SingleFileBackingStore { } - protected override byte[] SerializeSnapshot(CultPersistedStoreSnapshot snapshot) + protected override byte[] SerializeStore(string formatVersion, IReadOnlyList catalogEntries, IReadOnlyList records) { - return CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot); + return CultDocumentMessagePackSerialization.SerializeStore(formatVersion, catalogEntries, records); } - protected override CultPersistedStoreSnapshot DeserializeSnapshot(byte[] data) + protected override byte[] SerializeRecord(CultPersistedRecord record) { - var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(data, CultDocumentMessagePackSerialization.RequireReadableSingleFileStamp); - CultDocumentMessagePackSerialization.RequireSingleFileFormat(snapshot); - return snapshot; + return CultDocumentMessagePackSerialization.SerializePersistedRecord(record); } - protected override byte[] SerializePayload(object document) + protected override byte[] SerializeCatalogEntry(CultSchemaCatalogEntry entry) { - return CultDocumentMessagePackSerialization.SerializeUntyped(document, document.GetType(), Registry); + return CultDocumentMessagePackSerialization.SerializeCatalogEntry(entry); + } + + protected override (CultPersistedStoreSnapshot Snapshot, byte[][] CatalogBytes, byte[][] RecordBytes) ReadStore(byte[] data) + { + var store = CultDocumentMessagePackSerialization.ReadStore(data, CultDocumentMessagePackSerialization.RequireReadableSingleFileStamp); + CultDocumentMessagePackSerialization.RequireSingleFileFormat(store.Snapshot); + return store; + } + + protected override byte[] SerializePayload(object document, Type documentType) + { + return CultDocumentMessagePackSerialization.SerializeUntyped(document, documentType, Registry); } protected override object DeserializePayload(Type documentType, byte[] payload) diff --git a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs index 11a84591..63180a1b 100644 --- a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs +++ b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs @@ -15,7 +15,9 @@ namespace GameCult.Caching.MessagePack; // A small hot manifest indexes the store; each record lives in one cold content-addressed page. public sealed class DirectoryMessagePackBackingStore : CacheBackingStore { - private const string IndexedFormatVersion = "cultcache.store.v4.directory-content-addressed-pages"; + // The two manifest headers; the shared header decision names them. + private static readonly string IndexedFormatVersion = HeaderFor(holdsIds: false, existingHeader: null, wholeStore: true, directoryStore: true); + private static readonly string IndexedFormatVersionWithIds = HeaderFor(holdsIds: true, existingHeader: null, wholeStore: true, directoryStore: true); // An unleased load that has not settled after this many attempts throws. private const int UnleasedLoadAttempts = 5; private readonly FileInfo _manifestFile; @@ -23,7 +25,6 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore private readonly ConcurrentDictionary _dirtyKeys = new(StringComparer.Ordinal); private readonly ConcurrentDictionary _deletedKeys = new(StringComparer.Ordinal); private readonly HashSet _hydratedKeys = new(StringComparer.Ordinal); - private CultSchemaCatalogEntry[] _durableCatalog = Array.Empty(); public DirectoryMessagePackBackingStore(string manifestPath, string? recordDirectoryPath = null, bool readOnly = false) : base(readOnly) @@ -62,18 +63,19 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore var reports = new List(); var loaded = new Dictionary(StringComparer.Ordinal); - CultSchemaCatalogEntry[] catalog; + var foreign = new List(); for (var attempt = 1; ; attempt++) { loaded.Clear(); reports.Clear(); + foreign.Clear(); // Only pages named by the manifest read under the lease are loaded; orphaned pages are never loaded. With no // lock to lease (none created yet, or a store copied without it) a writer may commit mid-load, so an unleased // load stands only if its manifest is still current; if the manifest moved it reloads, under the lease the // writer created. A page that fails under an unmoved manifest is corruption, not a race. using var lease = AcquireCommitLease(wait: true, create: false); var manifestBytes = ReadManifestBytes(); - var manifest = ParseManifest(manifestBytes); + var (manifest, _) = ParseManifest(manifestBytes); Trace($"manifest records={manifest.Records.Length}"); try { @@ -83,10 +85,10 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore manifest.Records.OrderBy(record => record.Key, StringComparer.Ordinal).ToArray(), manifest.SchemaCatalog, loaded, - reports); + reports, + foreign); if (lease != null || !ManifestMoved(manifestBytes)) { - catalog = manifest.SchemaCatalog; Trace($"indexed-pages loaded={loaded.Count}"); break; } @@ -109,8 +111,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore var arrived = loaded .Where(pair => !Staged(pair.Key) && (!Entries.TryGetValue(pair.Key, out var existing) || - existing.StoredAt != pair.Value.StoredAt || - existing.PersistedSchemaId != pair.Value.PersistedSchemaId)) + !SameStoredIdentity(existing.StoredVersion, pair.Value.StoredVersion))) .Select(pair => pair.Value) .ToArray(); var departed = _hydratedKeys @@ -121,7 +122,6 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore Loaded?.Invoke(arrived, departed); // Adopted only once the cache admitted the load; a refused load leaves the store's durable view untouched. - _durableCatalog = catalog; foreach (var stored in departed) Entries.TryRemove(stored.Key.Value, out _); _hydratedKeys.RemoveWhere(key => !loaded.ContainsKey(key) && !Staged(key)); @@ -131,6 +131,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore _hydratedKeys.Add(key); SetLastSchemaMigrationReports(reports); + SetForeignRecords(foreign); IsDirty = !_dirtyKeys.IsEmpty || !_deletedKeys.IsEmpty; Trace("publish"); } @@ -179,10 +180,19 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore using var commitLease = AcquireCommitLease(wait, create: true); if (commitLease == null) return CultCommitOutcome.Contended; - var manifest = ReadManifest(); - if (!request.ConditionsHold(manifest.Records, Entries.Values)) + var (manifest, catalogBytes) = ReadManifest(); + if (request.HasConditions && !request.ConditionsHold( + manifest.Records.ToDictionary(record => record.Key, record => VersionOf(record.Payload), StringComparer.Ordinal), Entries.Values, ForeignRecords)) return CultCommitOutcome.Mismatch; + RefuseUndeclaredOnDisk(manifest, request.Upserts.Concat(request.Deletes).Select(entry => entry.Key.Value).Concat(request.Checks).Concat(StagedKeys)); + if (request.Upserts.Count == 0 && request.Deletes.Count == 0 && !IsDirty) + { + // Nothing to write: the batch only judged keys it does not hold. + MarkFlushSucceeded(); + return CultCommitOutcome.Committed; + } + var previousEntries = Entries.ToDictionary(pair => pair.Key, pair => pair.Value, StringComparer.Ordinal); var previousDirtyKeys = _dirtyKeys.Keys.ToArray(); var previousDeletedKeys = _deletedKeys.Keys.ToArray(); @@ -202,7 +212,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore _deletedKeys.TryRemove(entry.Key.Value, out _); } IsDirty = true; - WriteGeneration(manifest); + WriteGeneration(manifest, catalogBytes); return CultCommitOutcome.Committed; } catch @@ -227,20 +237,81 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore ThrowIfReadOnly(); Held(() => { - if (!IsDirty) + if (!IsDirty && StagedChecks.Count == 0) return; Directory.CreateDirectory(_manifestFile.DirectoryName!); using var commitLease = AcquireCommitLease(wait: true, create: true); - WriteGeneration(ReadManifest()); + var (manifest, catalogBytes) = ReadManifest(); + RefuseUndeclaredOnDisk(manifest, StagedKeys); + if (IsDirty) + WriteGeneration(manifest, catalogBytes); + else + MarkFlushSucceeded(); }); } + // Every key the next flush or commit judges: the staged writes and removals, and the staged checks. + private IEnumerable StagedKeys => _deletedKeys.Keys.Concat(_dirtyKeys.Keys).Concat(StagedChecks); + + // A writable store judges a removal of a key its cache does not hold at its next flush or commit, under the commit lease + // (StageChecks); a read-only store has neither, so it answers from its view and never creates the lease. + protected internal override void RefuseRemovalOfUndeclared(IReadOnlyCollection keys) + { + if (IsReadOnly) + base.RefuseRemovalOfUndeclared(keys); + else + StageChecks(keys); + } + + // A read-only store answers a batch's checks from its view and never creates the lease; a writable one reads the manifest + // under the commit lease. + protected internal override CultCommitOutcome JudgeChecks(IReadOnlyCollection keys, bool wait) + { + if (IsReadOnly) + return base.JudgeChecks(keys, wait); + return Held(() => + { + using var commitLease = AcquireCommitLease(wait, create: true); + if (commitLease == null) + return CultCommitOutcome.Contended; + RefuseUndeclaredOnDisk(ReadManifest().Snapshot, keys); + return CultCommitOutcome.Committed; + }); + } + + // A manifest record no registered type claims, listed at the version its index record names. + private CultForeignRecord? ForeignAt(CultPersistedRecord record, IReadOnlyCollection catalog) => + Foreign(record, catalog, VersionOf(record.Payload)); + + // The manifest's other records stay as they are, foreign and read-only ones included; a write that would replace or remove a + // record the manifest holds that the writer does not own is refused before any page is written, by the one rule every + // store shares (CacheBackingStore.RefuseUndeclaredOnDisk). The refused records reload as a pull loads them. + private void RefuseUndeclaredOnDisk(CultPersistedStoreSnapshot currentManifest, IEnumerable written) + { + CultPersistedRecord? HeldAt(string key) => + currentManifest.Records.FirstOrDefault(record => string.Equals(record.Key, key, StringComparison.Ordinal)); + RefuseUndeclaredOnDisk( + written, + key => HeldAt(key) is { } held ? Undeclared(held, currentManifest.SchemaCatalog, Registry.Owns) : null, + key => HeldAt(key) is { } held && Foreign(held, currentManifest.SchemaCatalog, VersionOf(held.Payload)) == null + ? LoadPage(held, currentManifest.SchemaCatalog, out _, out _) + : null, + key => HeldAt(key) is { } held ? ForeignAt(held, currentManifest.SchemaCatalog) : null, + key => + { + _dirtyKeys.TryRemove(key, out _); + _deletedKeys.TryRemove(key, out _); + IsDirty = !_dirtyKeys.IsEmpty || !_deletedKeys.IsEmpty; + }); + } + // Runs under the commit lease: pages first, then the manifest that names them. - private void WriteGeneration(CultPersistedStoreSnapshot currentManifest) + private void WriteGeneration(CultPersistedStoreSnapshot currentManifest, IReadOnlyDictionary catalogBytes) { Directory.CreateDirectory(_recordDirectory.FullName); var currentIndex = currentManifest.Records.ToDictionary(record => record.Key, record => record, StringComparer.Ordinal); + var replaced = currentManifest.Records.ToDictionary(record => record.Key, record => record.StoredAt, StringComparer.Ordinal); foreach (var key in _deletedKeys.Keys) currentIndex.Remove(key); foreach (var key in _dirtyKeys.Keys) @@ -249,21 +320,9 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore currentIndex[key] = ToIndexRecord(stored); } - var catalogCandidates = currentManifest.SchemaCatalog - .Concat(_durableCatalog) - .Concat(Entries.Values.Select(entry => entry.Descriptor.ToCatalogEntry())) - .GroupBy(entry => entry.SchemaId, StringComparer.Ordinal) - .Select(group => group.Last()) - .ToArray(); - var usedSchemaIds = currentIndex.Values.Select(record => record.SchemaId).ToHashSet(StringComparer.Ordinal); - var targetCatalog = catalogCandidates - .Where(entry => usedSchemaIds.Contains(entry.SchemaId)) - .OrderBy(entry => entry.SchemaName, StringComparer.Ordinal) - .ToArray(); var keysToWrite = _dirtyKeys.Keys .OrderBy(value => value, StringComparer.Ordinal) .ToArray(); - var written = new List(keysToWrite.Length); foreach (var key in keysToWrite) { if (!Entries.TryGetValue(key, out var stored)) @@ -271,6 +330,9 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore continue; } + // A record this write replaces is stored later than it: the one re-mint every store kind shares (StoredAtOver). + if (replaced.TryGetValue(key, out var replacedAt)) + stored.StoredAt = StoredAtOver(stored.StoredAt, replacedAt); var record = ToPersistedRecord(stored, document => CultDocumentMessagePackSerialization.SerializeUntyped( document, @@ -278,45 +340,76 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore Registry)); var pagePayload = CultDocumentMessagePackSerialization.SerializePersistedRecord(record); var indexRecord = ToContentAddressedIndexRecord(record, pagePayload); + stored.StoredVersion = VersionOf(indexRecord.Payload); + stored.StoredSchemaId = stored.Descriptor.SchemaId; currentIndex[key] = indexRecord; WriteFileAtomically( ContentAddressedRecordPath(indexRecord), pagePayload); - written.Add(stored); } - WriteManifest(targetCatalog, currentIndex.Values + // The catalog and the header follow the shared write rules: the registered entry publishes only the ids this write put a record + // under, every other record and entry stays as the manifest holds it. + var written = keysToWrite.Where(key => Entries.ContainsKey(key)).ToHashSet(StringComparer.Ordinal); + var targetCatalog = CultSchemaCatalogEntry.Derive( + currentIndex.Values, + written.Select(key => Entries[key].Descriptor).Distinct().Select(descriptor => descriptor.ToCatalogEntry()).ToArray(), + currentManifest.SchemaCatalog) + .OrderBy(entry => entry.SchemaName, StringComparer.Ordinal) + .ThenBy(entry => entry.SchemaId, StringComparer.Ordinal) + .ToArray(); + + var holdsIds = written.ToDictionary(key => key, key => Entries[key].HoldsIds, StringComparer.Ordinal); + var header = HeaderForWrite( + currentManifest.FormatVersion, + holdsIds.Values.Select(holds => (bool?)holds).ToArray(), + copiesAny: currentIndex.Values.Any(record => !written.Contains(record.Key)), + copiedHoldIds: false, // only C# writes a directory store, and it marks by content + directoryStore: true, + holdsVariants: false); + WriteManifest(targetCatalog, catalogBytes, currentIndex.Values .OrderBy(record => record.Key, StringComparer.Ordinal) - .ToArray()); - // The manifest names the new pages: each was written under its own descriptor. - foreach (var stored in written) - MarkWritten(stored); + .ToArray(), header); DeleteUnreferencedRecordPages(currentIndex.Values); - _durableCatalog = targetCatalog; _dirtyKeys.Clear(); _deletedKeys.Clear(); MarkFlushSucceeded(); } - private void WriteManifest(CultSchemaCatalogEntry[] catalog, CultPersistedRecord[] index) - { - var manifest = new CultPersistedStoreSnapshot - { - FormatVersion = IndexedFormatVersion, - SchemaCatalog = catalog, - Records = index - }; - WriteFileAtomically(_manifestFile.FullName, CultDocumentMessagePackSerialization.SerializeSnapshot(manifest)); - } + // The index records are this store's own and are encoded from their fields; a catalog entry the manifest already held is written + // as the slice it held, one the writer built is encoded. + private void WriteManifest( + CultSchemaCatalogEntry[] catalog, + IReadOnlyDictionary catalogBytes, + CultPersistedRecord[] index, + string header) => + WriteFileAtomically( + _manifestFile.FullName, + CultDocumentMessagePackSerialization.SerializeStore( + header, + catalog.Select(entry => catalogBytes.TryGetValue(entry, out var held) ? held : CultDocumentMessagePackSerialization.SerializeCatalogEntry(entry)).ToArray(), + index.Select(CultDocumentMessagePackSerialization.SerializePersistedRecord).ToArray())); private void LoadRecordPages( CultPersistedRecord[] records, IReadOnlyCollection catalogEntries, Dictionary loaded, - List reports) + List reports, + List foreign) { + // A record no registered type claims stays as the manifest names it: its page is never read, and a commit keeps it. + var claimed = new List(records.Length); + foreach (var metadata in records) + { + if (ForeignAt(metadata, catalogEntries) is { } carried) + foreign.Add(carried); + else + claimed.Add(metadata); + } + + records = claimed.ToArray(); var tracePages = string.Equals( Environment.GetEnvironmentVariable("CULTCACHE_TRACE_STARTUP_PHASES"), "1", @@ -342,16 +435,10 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore index => { var started = tracePages ? Stopwatch.GetTimestamp() : 0L; - var metadata = records[index]; - var record = ReadPersistedRecordPage(metadata, out var pagePayload); - storedRecords[index] = ToStoredDocument( - record, - catalogEntries, - (type, payload) => CultDocumentMessagePackSerialization.DeserializeUntyped(type, payload, Registry), - out recordReports[index]); + storedRecords[index] = LoadPage(records[index], catalogEntries, out recordReports[index], out var pageLength); if (tracePages) { - pageBytes[index] = pagePayload.LongLength; + pageBytes[index] = pageLength; pageElapsedTicks[index] = Stopwatch.GetTimestamp() - started; } }); @@ -409,7 +496,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore Payload = HashPayload(pagePayload) }; - private CultPersistedStoreSnapshot ReadManifest() => ParseManifest(ReadManifestBytes()); + private (CultPersistedStoreSnapshot Snapshot, Dictionary CatalogBytes) ReadManifest() => ParseManifest(ReadManifestBytes()); private byte[]? ReadManifestBytes() => File.Exists(_manifestFile.FullName) ? ReadAllBytesShared(_manifestFile.FullName) : null; @@ -435,37 +522,51 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore private InvalidOperationException Unsettled(Exception? last) => new( $"Directory store {_manifestFile.FullName} changed under every one of {UnleasedLoadAttempts} unlocked loads; it did not settle.", last); - // A missing manifest is an empty store. An existing manifest in any other format is refused. - private CultPersistedStoreSnapshot ParseManifest(byte[]? manifestBytes) + // A missing manifest is an empty store. An existing manifest in any other format is refused. Beside the snapshot, the bytes the + // manifest held for each catalog entry, by the entry this read produced: the slices a write carries forward. + private (CultPersistedStoreSnapshot Snapshot, Dictionary CatalogBytes) ParseManifest(byte[]? manifestBytes) { if (manifestBytes == null) { - return new CultPersistedStoreSnapshot + return (new CultPersistedStoreSnapshot { FormatVersion = IndexedFormatVersion, SchemaCatalog = Array.Empty(), Records = Array.Empty() - }; + }, new Dictionary()); } - // The stamp is judged before the manifest body is parsed, so a store stamped newer is refused whatever its body. - return CultDocumentMessagePackSerialization.DeserializeSnapshot(manifestBytes, RequireReadableStamp); + // The stamp is judged before the manifest body is parsed, so a store stamped newer is refused whatever its body. Every + // write path parses the manifest under its commit lease before it writes a page or the manifest, so a store another + // process has since stamped newer is refused there too, untouched. + (CultPersistedStoreSnapshot Snapshot, byte[][] CatalogBytes, byte[][] RecordBytes) read; + try + { + read = CultDocumentMessagePackSerialization.ReadStore(manifestBytes, RequireReadableStamp); + } + catch (CultStoreUnreadableException ex) when (ex.Path == null) + { + throw new CultStoreUnreadableException(ex.Message, _manifestFile.FullName, ex.InnerException); + } + + var snapshot = read.Snapshot; + // An array with no stamp claims no directory format: it parses to the default v1 without the callback above ever seeing a stamp. + RequireReadableStamp(snapshot.FormatVersion); + var catalogBytes = new Dictionary(); + for (var index = 0; index < snapshot.SchemaCatalog.Length; index++) + catalogBytes[snapshot.SchemaCatalog[index]] = read.CatalogBytes[index]; + return (snapshot, catalogBytes); } + // The one verdict on a directory manifest's stamp: this runtime reads v4 and v5. private void RequireReadableStamp(string stamp) { - if (CultPersistedStoreSnapshot.FormatNumber(stamp) is int number && number > 4) + if (!string.Equals(stamp, IndexedFormatVersion, StringComparison.Ordinal) && + !string.Equals(stamp, IndexedFormatVersionWithIds, StringComparison.Ordinal)) { - // Every write path parses the manifest under its commit lease before it writes a page or the manifest, so a - // store another process has since stamped newer is refused here, untouched. - throw new NotSupportedException( - $"Directory store {_manifestFile.FullName} is {stamp}, a newer format than this runtime reads ({IndexedFormatVersion})."); - } - - if (!string.Equals(stamp, IndexedFormatVersion, StringComparison.Ordinal)) - { - throw new InvalidOperationException( - $"Directory store {_manifestFile.FullName} is {stamp}; only {IndexedFormatVersion} is readable."); + throw new CultStoreUnreadableException( + $"Directory store {_manifestFile.FullName} is {CultDocumentMessagePackSerialization.DescribeHeader(stamp)}; only {IndexedFormatVersion} and {IndexedFormatVersionWithIds} are readable.", + _manifestFile.FullName); } } @@ -507,6 +608,26 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore } } + // One record as a load decodes it, from the page its manifest record names: a pull's pages and a refusal's reload. + private CultStoredDocument LoadPage( + CultPersistedRecord metadata, + IReadOnlyCollection catalog, + out CultSchemaMigrationReport report, + out long pageLength) + { + var record = ReadPersistedRecordPage(metadata, out var pagePayload); + pageLength = pagePayload.LongLength; + var stored = ToStoredDocument( + record, + VersionOf(metadata.Payload), + catalog, + (type, payload) => CultDocumentMessagePackSerialization.DeserializeUntyped(type, payload, Registry), + out report); + // The manifest is this store's record of what is on disk: the stored id and the version a commit condition compares with. + stored.StoredSchemaId = metadata.SchemaId; + return stored; + } + private CultPersistedRecord ReadPersistedRecordPage(CultPersistedRecord metadata, out byte[] pagePayload) { var path = ContentAddressedRecordPath(metadata); diff --git a/src/GameCult.Caching.MessagePack/GameCult.Caching.MessagePack.csproj b/src/GameCult.Caching.MessagePack/GameCult.Caching.MessagePack.csproj index b81dcf92..c2588e24 100644 --- a/src/GameCult.Caching.MessagePack/GameCult.Caching.MessagePack.csproj +++ b/src/GameCult.Caching.MessagePack/GameCult.Caching.MessagePack.csproj @@ -11,6 +11,5 @@ - diff --git a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md index 57b5d7fb..10fb9fc8 100644 --- a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md +++ b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md @@ -46,23 +46,71 @@ store. The store commits the batch as one durable step. Nothing in the batch is visible, to the committing flow or to observers, until the store has accepted it; a failed commit changes nothing on disk or in memory. -A batch may carry conditions: per-record `(schemaId, storedAt)` identity, or -the whole store unchanged since it was last loaded. Conditions are evaluated +A batch may carry conditions: per-record version, or the whole store unchanged +since it was last loaded. A record's version is the SHA-256 of the exact bytes the +record is stored as: in a single file the record's own slice of the file, the array +`[key, schemaId, storedAt, payload(, variant)]` exactly as the file holds it, in +whatever legal MessagePack encoding its writer chose (never a re-encoding of the +decoded record); in a directory store the page the manifest indexes. A record being +written has the version of the bytes it is written as. It names the schema id and +storedAt as well as the payload. storedAt is metadata and decides no condition. +Conditions are evaluated under the store's exclusive lock against what is durably on disk. A failed condition is a lost race, not an error: the commit reports it and writes nothing. -**Conditional commit is the only safe multi-process write.** A plain flush and -an unconditional commit are last-writer-wins and write the same bytes: a single -file is replaced by the writer's whole view, a directory manifest takes the -writer's staged keys over the current manifest. An unconditional commit also -persists writes staged earlier and leaves the store clean. Processes sharing a -store must all use conditional commit. +**A flush, a commit and a single-file document write apply a write set.** Under +the store's exclusive lock the writer reads what is durably in the store now, +applies only what it staged (writes and removals) and, for a commit, its batch, +and replaces the store with the result. Every other record, and every catalog +entry the result references, is copied forward as stored: a record's exact bytes +and a catalog entry's exact bytes, unknown fields included, so a copied record +keeps its version. No type interprets a copied record: a write may hash another +record's bytes to compare versions, and reads its header (key, schema id, variant +slot), but never decodes its payload. +A record is encoded from the object the writer hands in, and every other record is copied as the store holds it. The +store that read a file owns that file's bytes: for each stored record, its exact slice of the file and the SHA-256 of that +slice, which is the record's version; likewise each catalog entry's slice. A decoded record or snapshot object carries no +bytes, so an object a caller edited in place is never mistaken for what the file holds. Consequently a record the writer +staged is re-encoded canonically and has the version of its new encoding, even when its content is unchanged, and a +record nobody staged keeps its bytes and its version for as long as no writer stages it. The public snapshot and record +serializers encode every record from its fields. +A record written over one the store holds gets a storedAt later than the one it +replaces. A document changed in place and not staged is not written. + +**In a store that holds no variant**, a plain flush and an unconditional commit +write the same bytes and are last-writer-wins only per staged key. An +unconditional commit also persists writes staged earlier and leaves the store +clean. Conditional commit is how such a writer learns that another writer moved +the keys it depends on. + +**A store that holds a variant, or would after the write, is written only from +what its writer last read.** A variant resolves against other records (its base +chain, and the index values it shares the store with). So before writing, the +writer compares every record's version with the version it last read +or wrote. If any record moved, appeared or disappeared, the whole write is +refused with the typed write conflict, naming the keys it would have written and +the keys that moved, and nothing is written. A writer that has read nothing, +such as CultMesh's single-file document write, never writes into a store holding +a variant. + +**A refused write reloads what it refused.** A refusal (a foreign record at a +key the write names, or a write conflict) reloads the durable view of every +refused and moved key into the cache, as a pull would, and only then forgets the +staged changes at those keys. The cache then serves what the store holds, and a +retry is judged against that. If the reload is itself refused (the store holds a +record this cache cannot load), nothing changes and the refusal carries that +cause. A writer refuses a file whose header it cannot read (truncated, or a +format it does not read) and leaves the file as it was. C# applies this to +single-file stores, directory stores and CultMesh's single-file document writes; +Rust, TypeScript and Python rewrite the store from their own copy until their +parity cuts land. A store that fails to hydrate on open (corrupt bytes, unresolvable schema) makes the open throw and is left byte-identical; it is never silently overwritten. Within one process the lock order is the cache's gate, then the -store's file lock; a store's I/O blocks its cache's readers. +store's gate, then the file lock; nothing under a store's gate enters a cache's +gate. A store's I/O blocks its cache's readers. C# implements batch and conditional commit (`CultCache.Commit`, `TryCommit`, `CultCacheBatch`). Rust implements both. TypeScript and Python implement @@ -116,6 +164,87 @@ These are store-level bookkeeping fields, not per-record domain metadata. The store must embed a catalog entry for every schema referenced by the records it contains. +A reader never recovers a schema from a record's payload: a record whose schema id the +catalog does not publish (by id or as a compatible id) makes the store unreadable, whatever +the payload's first field says. The catalog is the store's own description of itself. + +A writer keeps the same invariant: every record it writes carries a schema id that some catalog +entry it writes publishes, as that entry's id or as one of its compatible ids. A cache stamps a +record it puts with its registered schema's id, and a record it loads under an id its type owns or +declares compatible (see below for one it does not). The cache is the only owner of that stamp: a store writer never +changes the identity of a record it did not receive from its caller, so a record another writer +changed since this cache read it is written back as that writer left it. + +A schema type declares the older ids its records may sit under: `CultDocumentAttribute.CompatibleSchemaIds` +(C#), `compatibleSchemaIds` (TypeScript, Python). The registered catalog entry lists them after its own id, and +the cache restamps a record loaded under one to the type's own id, so the next write of that record carries the +registered id. A record another writer left under a declared id, and that a commit does not rewrite, stays under it +and the registered entry publishes it. Without the declaration such a record is read-only to the cache (see +ownership by declaration, below). Rust has no such declaration. + +A cache resolves a record to a local type by the record's schema id: the registered type that owns +the id, else one that lists it as compatible. The name the catalog entry carries is metadata. It +names a local type only when no local type has the id, and then only to read with: a record so +resolved is read-only (below). That serves a store written by a runtime whose schema ids this one +cannot know (a Rust type's schema id is its entry type). A schema renamed under a stable id +therefore opens in every runtime that holds the id. + +**A cache owns a record when it declares the record's schema id, or when the type that reads it reads +it whole.** A type declares an id by registering it or listing it compatible. A type reads a record +whole when the catalog entry that publishes the id has the same content hash as the type's entry, or +compares compatible with it and leaves no persisted slot unread. An owned record is writable: a write, +relabel or removal is accepted and the write stores it under the writing type's own id (restamp on +write; only a record the cache writes is restamped, the rest is copied forward as stored). A record +stored under any other id that a type can read, through the catalog's schema name or an entry listing a +local id, only by dropping persisted slots is loaded read-only: Get and queries serve it, a write of +another record copies it as stored, and a staged write or removal of it (upsert, variant upsert, +flatten, remove, commit) is refused with the typed schema conflict before anything is staged. +Declaring its id compatible on any registered type makes it writable and is the consent to the loss: +the next write stores it under the writing type's own id and sheds the slots. + +The rule is checked again where the write lands, under the store's lock: a write or removal at a key +whose durable record is not owned, because another writer rewrote it after this cache +loaded, is refused typed, and the record is reloaded as the store holds it, read-only when a type can +read it. This holds for every write through the store, the store's own `Push`, `Delete` and +`CommitBatch` included. A CultMesh single-file write (`WriteDocument`, `WriteRecord`, or `PushAll` on an unattached store) asks the same rule: it owns the ids its +written entries declare (their own ids and the ids they list compatible) and an id whose publishing entry its claimant reads whole (no persisted +slot unread). The claimant is picked by the one precedence over the written entries and the call's registry, a written entry being the only +description of its schema id (the registry's type for that id is never asked), and the record is owned only when the claimant is a written +entry. A registry adds claimants and declares nothing, so it only adds refusals; with none the written entries are the only claimants. Refused: +a record another registered type claims, under that type's current id or an old one it lists; a record whose stored entry lists another +registered type's id ahead of the written one's, whatever name the stored entry carries; a record two registered types list compatible or two +registered versions of the written schema could read; a record the picked written entry reads only by dropping persisted slots; and the +removal of any stored record by a call that writes no entry. A registry-less `WriteDocument` reads `CultDocumentRegistry.Shared`. It is +refused the same way otherwise. + +A record that resolves to no local type is a foreign record, for example one of a type this build does not +have, or one renamed under a new schema name without declaring its old id. A store never destroys or relabels it: the store carries it +byte for byte under its own id and its own catalog entry, lists it as foreign (`CacheBackingStore.ForeignRecords` +in C#), and the cache never holds it. Writes of other records proceed and copy it forward as the file holds it. A write that would replace or remove it is refused with the typed schema conflict +naming its key and id, and nothing is written. A removal of a key the cache does not hold removes nothing and is free; the store judges the key at its next flush or commit, under its lock, with the keys that write writes, and refuses the write if it holds a record there under an undeclared id, also when that record arrived after the load. A commit judges such a key in every writable store its cache attaches, each under its own lock: the stores it does not write first, then the store it writes, and a refusal in any of them writes nothing. A judged key is never removed. A read-only store answers at the removal, from its last load. The refused keys are reloaded as the store holds them (a foreign record is not held, and is listed as foreign from then on, so its next write is refused before staging), and their staged changes are forgotten. A write that carries a record forward keeps a marked header marked. Declaring its id on a type claims it at the next load. + +A reader resolves a record's schema id to the entry that owns it, that entry's own `schemaId`, and +only when no entry owns the id to an entry that lists it as a compatible id (the first such entry +in catalog order). + +A writer derives the catalog from the records it writes, and from nothing else. For each distinct +schema id a record carries it chooses exactly one entry that publishes it. For an id the write stages +a record under, that is the entry the writer registered or passed, and a rename or a stale content hash +takes the descriptor's name, hash, members and compatible ids as they are. For every other id it is +the durable entry that owns the id, else one that lists it as compatible, carried as its exact bytes, +unknown trailing fields included. A registered entry never publishes an id the write stages nothing +under. Entries are written as chosen, deduplicated by own id, with +no union of compatible ids and no dependence on the order records arrive in; an entry no record needs +is not written. A write is refused, and the store left as it was, when two entries of one tier share +an own id and disagree on the schema name, when records of different types share a schema id, when a +write would change which type an existing record resolves to (the schema names of the records already +in the store and of the records written differ), or when a record's id is published by no chosen entry. +The retyping refusal has one exception, a rename: when the entries of both the existing records and +the written ones own the id, the registered descriptor wins. (A C# cache stamps every record it +writes with its descriptor's own id, so the retyping refusal cannot arise there.) Each runtime refuses with one typed error naming the id, the +schema names and a record key: `CultSchemaConflictException` (C#), `SchemaConflictError` +(Rust, TypeScript, Python). + This is not decorative paperwork. It is what allows another CultCache implementation to inspect an old store, resolve the schema for each record, and decide whether it can: @@ -212,10 +341,46 @@ Example: The payload is the domain object only. It does not repeat the key, schema id, or stored timestamp. -Under `cultcache.store.v1` a record has exactly these four slots. A reader refuses any -`cultcache.store.*` header it does not know and any record with more slots than its -header allows, naming the version or the record's key and schema id. It never skips a -slot it does not understand. +Under `cultcache.store.v1` and `cultcache.store.v3` a record has exactly these four slots. +A reader refuses any `cultcache.store.*` header it does not know and any record with more +slots than its header allows, naming the version or the record's key and schema id. It +never skips a slot it does not understand. + +The headers are `v1` (no variant, no element ids), `v2` (holds a variant) and `v3` (holds +an element id, variants or not). A single-file store declares `v3` when any record it +holds carries an element id: a non-empty `[CultElementId]` value in a record's document, +or in a variant's override values. A store that cannot read a variant's override values +(no codec attached) counts the variant as holding one. An element id is an ordinary keyed +string in the payload, so only a reader with the record's type can tell; a C# write +decides for the records it encodes and, while the header it found is unmarked, for every +plain record it copies forward that its registry can read, decoding the bytes the file +holds under the store's lock. A copied record it cannot read, and a raw CultMesh payload, +decide nothing. A write that copies any record forward +therefore keeps a `v3` header it found. Only a write that encodes every record the store +will hold decides by content, and drops to `v1` (or `v2` with a variant) when none holds +an id. A directory store's manifest says +`cultcache.store.v5.directory-content-addressed-pages` under the same rule, and `v4` +otherwise. + +A single-file store is exactly one MessagePack array of three slots: the header, the schema +catalog and the records. Every runtime refuses a file that is truncated, has bytes after the +array (a second store appended included), is not an array, has fewer or more than three slots +(a header alone included), whose first slot is not a header it reads, or whose catalog or +records it cannot decode (a record with more slots than its header allows, a variant record +it does not read). A runtime never skips a slot it does not understand. It refuses on open +and on rewrite alike, because the rewrite asks the same full reader as the open. Only nothing at the path is a missing +file, and a missing file and an empty array are an empty store; a zero-byte file is not a store and is refused as unreadable. +In C#, a link whose target is gone and a directory at the path are I/O errors, never an empty store. Within a runtime, every +reader and writer of a single-file store shares that one reader, lock and replace (in C#, CultMesh's single-file document +helpers included), and a catalog entry has one layout. A flush or a commit replaces a file only when the +runtime's own reader opens it, so one verdict covers open, flush and commit, and a file the +runtime cannot read is left as it is. Each runtime refuses with one typed error: +`CultStoreUnreadableException` (C#), `CultCacheStoreUnreadable{path, kind}` (Rust, its kind +telling an undecodable store from an unsupported format), `StoreUnreadableError` (TypeScript, +Python), with the underlying cause attached. Rust, TypeScript and Python also open the legacy envelope +array (an array of `key`/`type`/`payload`/`storedAt` maps) and replace it; C# has no legacy +reader and refuses it. The shared byte vectors and each runtime's verdict are +`tests/vectors/document-variants-c2a/readability/manifest.txt`. A record's payload nests at most 256 MessagePack containers. Every array and map counts, whether a reader decodes it or skips past it, and the outermost array is @@ -243,17 +408,73 @@ fifth slot: `elementId` is `""` unless the step enters an object-list element. `value` is the member's own MessagePack encoding, inline. A top-level Set has one step and `id` `""`; a reader that resolves only that shape refuses the others, naming the variant. -- A store holding at least one variant declares `cultcache.store.v2`; a store holding - none declares `cultcache.store.v1` and is byte-identical to what a runtime without - variants writes. A `v1` header over a variant record is refused. +- A store holding at least one variant declares `cultcache.store.v2` (`v3` when it holds an + element id); a store holding none declares `cultcache.store.v1` (or `v3`) and is + byte-identical to what a runtime without variants writes. A `v1` header over a variant + record is refused. - A variant's resolved document is derived in memory when the cache admits the record or any record on its base chain, and is never written back. A plain write at a variant's key is refused; `Flatten` replaces a variant with a plain record holding its resolved document. -- The C# runtime resolves variants. Every other runtime refuses a `v2` store by name. +- The C# runtime resolves variants. Every other runtime refuses a `v2` store by name, and + refuses a variant record inside a `v3` store by its record key. CultMesh single-file reads, the directory store and CultNet refuse a variant record by name until each has its own cut. +## Element Ids + +Every object that sits in a list, array or dictionary value of a shape the element-id walk +owns carries an id: one string member with an integer `[Key]` and `[CultElementId]`. The +owned shapes are arrays, `IEnumerable` classes and interfaces, `IDictionary<,>` and +`IReadOnlyDictionary<,>` values, and `[MessagePackObject]` or `[Union]` types with integer +keys (a keyed struct included), read through the members MessagePack writes. Union subtypes +are included. The id identifies the element inside its list, so an override can name it (a +path step is `[slot, elementId]`) and a base edit that reorders or inserts elements +retargets nothing. + +Any other shape is one whole value. Its elements carry no element ids: a `[CultElementId]` +member inside it is a plain string, never minted or checked and never counted for `v3`. A +variant replaces it whole (operator, 2026-10-16). + +- A random id is 12 lowercase hex characters. A derived id + (`[CultElementId(nameof(Source))]`, for content-addressed elements) is the + invariant-culture text of its source member, which must be non-empty; it needs no + other format. An id is unique within its list. +- A write mints random ids for unset elements, including inside variant override + values, and refuses a duplicate, a malformed random id or an empty derived source. A + load refuses the same things, naming the record, the list and the id. A record + written before ids existed loads, and its ids are minted from SHA-256 of the record + key and the element's path, so every reader mints the same ids; they reach disk with + the record's next write, or `CultCache.MintElementIds()`. +- A store that holds an id declares the `v3` header (above), so a reader older than ids + refuses it instead of skipping the id slot and rewriting the elements without ids. + A C# write also marks the store when a record it copies forward holds an id, judged + from the file's bytes at the write (above). TypeScript, + Python and Rust read `v3` and keep a `v3` header they found when they rewrite a + store; they carry ids and do not decide when a document carries them. +- An object list's elements are written by their `[Key]` layout, because the id is a + keyed member of the element. Value lists are replaced whole: a list of a non-keyed + class or struct, or of primitives. So is a list of keyed structs, since a struct is a + value and never an element, but an object list inside a keyed struct is walked like any + other. The walk follows declared member types: a member declared `object` is untyped on + disk, and a subclass's extra members under a non-union base-typed member are not written, + so neither is walked. Whole values are `ImmutableArray`, `ArraySegment`, + `Memory`, `KeyValuePair`, tuples, `Lazy`, a class or struct without + `[MessagePackObject]`, a `[DataContract]` type, a string-keyed type, a type or member + declaring `[MessagePackFormatter]`, and a non-public member of a type without + `AllowPrivate`. The members read are an integer `[Key]` with no `[IgnoreMember]`: a + public field or a property whose getter is public, or any visibility up the base chain + under the most derived `[MessagePackObject(AllowPrivate = true)]` declaration. An id member + needs a setter MessagePack calls (public or `init`; any under `AllowPrivate`), so the id + MessagePack writes is read back. Still refused: a bad id member (missing, duplicate, not + a string, no integer `[Key]`, `[MessagePackFormatter]` on it, one MessagePack does not + write or cannot assign, a key shared with another member, a bad derived source), a + concrete union arm that is not keyed, a keyed type that is also a collection + (`ICollection`, `IList`, `IDictionary`: MessagePack writes it as a collection, so C# + refuses it at the first encode or decode when it is on the path to an object list), a + type MessagePack has no formatter for, and a formatter resolver that claims a type on the + path. A formatter MessagePack's source generator emits is not refused. + ## Canonical Semantic Schema Hash `schemaId` should be derived from a canonical semantic schema hash. @@ -408,8 +629,12 @@ The v1 concurrent single-file policy is: 2. Writers take an exclusive sidecar lock derived from the `.cc` path. 3. Writers re-read the current snapshot after taking the lock. 4. Writers using conditional commit evaluate their conditions against the - latest snapshot and write nothing if one fails. A plain flush and an - unconditional commit compare nothing and are last-writer-wins. + latest snapshot and write nothing if one fails. In a store with no variant, a + plain flush and an unconditional commit compare nothing: they apply their + staged writes onto the current snapshot, last-writer-wins per staged key. A + write into a store that holds or would hold a variant compares every record's + version with what the writer last read, and is refused if any moved. One whose + header they cannot read is refused untouched. 5. Writers write a temp file, flush it, atomically replace the `.cc` file, and release the lock. diff --git a/src/GameCult.Caching/Contracts/cultcache-schema-compatibility.md b/src/GameCult.Caching/Contracts/cultcache-schema-compatibility.md index f44ff667..dc7e7bea 100644 --- a/src/GameCult.Caching/Contracts/cultcache-schema-compatibility.md +++ b/src/GameCult.Caching/Contracts/cultcache-schema-compatibility.md @@ -61,6 +61,12 @@ either way is refused: a get-only property, a readonly field without `AllowPrivate`, or a property whose setter is non-public without `AllowPrivate`, when the constructor does not fill it. +Never reuse a dropped slot; retire it. A write set copies a record the writer did not touch forward as stored bytes, so an +untouched record keeps a dropped member's value in its slot through every write. A type that later puts a new member in that slot +reads the dropped value as its own. The store does not remember retired slots, and a reuse that changes the slot's shape is refused +at load, but a reuse of the same shape reads silently. This is the rule protobuf, Thrift and MessagePack-CSharp give for numbered +fields: leave the slot empty, and take the next unused number for a new member. + ## Which schema a record carries A record is written under the schema of its runtime type. Writing a derived @@ -123,10 +129,15 @@ lie about the payload: - target schema change - name/index lookup semantics change - missing embedded catalog entry -- no compatible local schema candidate This is not negotiable. Better a loud refusal than quiet bit-rot. +## Foreign records + +A record with no local schema candidate is not rejected: the store carries it untouched as a +foreign record and refuses any write that would replace or remove it. See "foreign record" in +`cultcache-persistence-format.md`. + ## Variant overrides A variant's override follows the same outcome as the same member on a plain record; there diff --git a/src/GameCult.Caching/Contracts/cultcache-store-composition.md b/src/GameCult.Caching/Contracts/cultcache-store-composition.md index e23d491f..1a03d4b1 100644 --- a/src/GameCult.Caching/Contracts/cultcache-store-composition.md +++ b/src/GameCult.Caching/Contracts/cultcache-store-composition.md @@ -81,9 +81,8 @@ which route wrote it. after subscribing with a `Sequence` at or below the read is already in the document and is dropped. CultMesh cache-record mirrors (`ObserveAsync`, `ReactiveAsync`, and `RefreshAsync` on both) do this. -- Not stale-protected: schema-alias handles (`AsSchemaAlias`) get no - `Sequence`; removals do not advance a mirror's applied `Sequence`, and the - mirrors do not surface removals. +- Not stale-protected: removals do not advance a mirror's applied `Sequence`, + and the mirrors do not surface removals. - `AddJournal` registers an ordered admission journal, for audit, replication or a derived index. The cache calls each journal under its gate, right after an admission is applied and before that admission is published, once per @@ -127,10 +126,39 @@ which route wrote it. ## Locking -- One lock order: the cache's gate, then the store's lock. A store attached to a - cache takes that cache's gate as its own lock, so a store's load callback into - the cache cannot take the two out of order. A store must not call `Loaded` - while holding its own lock outside the cache's hold. +- One lock order: the cache's gate, then the store's gate, then the file lock. A + store's gate may be taken inside a cache's gate (the attach; a journal writing + a cacheless file), never the reverse. What the code enforces, at run time and + by `InvalidOperationException`: a thread holds at most one store's gate; + while it holds a store's gate it enters no cache gate it does not already + hold, reads included; a cacheless write made while the thread already holds + that store's gate is refused; and a cache's hold (a write, a flush, an attach) + entered inside another cache's hold is refused. What it does not enforce: a cache + read made inside another cache's hold takes that cache's gate without a + check, so two threads each holding one cache and reading the other can + deadlock. That is open as `variants:follow_up:cross-cache-read-lock-order`; + until it lands, a journal or a getter must not read a second cache. User code + a cacheless call or an attach runs under the store's gate (a document's + getter, a constructor while a copied record is read) is the way into the + refused cases. +- One owner picks the gate a store call holds. A store attached to a cache takes + that cache's gate as its own lock, so a store's load callback into the cache + cannot take the two out of order; a cacheless store takes its own gate. The + store reads which one applies, takes it, and checks the answer again under it: + `Cache` is written only by the attach, holding both gates: it publishes `Cache` + before its pull and takes it back in the same hold if the pull fails. Another + thread's unlocked guess may read the tentative cache, but none runs a body + against it: the guess is checked under that cache's gate, which the attach + holds, and fails once the pull retracts it. Under the store's gate `Cache` is + settled, and a settled cache never changes, because nothing detaches. A call + takes at most three gates: the guessed cache's, the store's, the settled + cache's. The pull reads the registry the attach lent, not `Cache`; a variant it + loads carries no codec, and the cache resolves it with its own. During an + attach the store is attached, so a journal the pull runs that calls the store + is refused as it would be afterwards; a cacheless write that finds a cache + published, during an attach or after it, is refused with the same text. A + store must not call `Loaded` while holding its own lock outside the cache's + hold. - A journal runs under the cache's gate, so a slow journal slows every writer of that cache. It must not enter the cache, and must not wait for anything that needs the cache or its gate: another thread's write, a lock a writer holds. @@ -140,8 +168,6 @@ which route wrote it. CultNet journal, under the cache's gate: it must not call the cache or wait for a thread that does. A store that writes its own cache throws through the journal's reentry guard, and the admission stands. -- A thread holds one cache's gate at a time: entering a cache's hold inside - another cache's hold throws. - A store doing I/O on behalf of its cache (pull, flush, commit) holds the gate, so that cache's readers wait for the I/O. The file lock (`.lock` or the directory commit lease) is always taken inside the gate and released before a @@ -223,26 +249,12 @@ against what is durably on disk at that moment: constrained; unrelated concurrent writes do not fail the commit. Passing any instance other than the one the cache holds throws. - `ExpectUnchanged()`: the store's persisted record set must equal what this - cache last loaded from it, compared as the ordered list of - `(key, schemaId, storedAt)`. Any insert, delete or replace fails it. + cache last loaded from it, compared as the ordered list of `(key, version)`. + Any insert, delete or replace fails it. -Record identity for conditions is `(schemaId, storedAt)`, where `schemaId` is -the id the record is persisted under (the internal `CultStoredDocument.PersistedSchemaId`), -not the one this cache decodes it as. A value change mints a `storedAt` -strictly later than the record it replaces; writers bump a minted timestamp by -one tick when it is not later. An entry loaded under compatible schema drift -keeps its persisted id until its store rewrites it; a store records each -record its completed write rewrote with the internal instance method -`CacheBackingStore.MarkWritten` (the shipped stores call it; it is not part of -the public API), which refuses an entry that is not the one -the store holds in its own `Entries` at that key. That guards against a store -marking by mistake an entry it never held; a store that writes another copy -into its own `Entries` and then marks it is trusted to have rewritten that -record, so a mark is the store's own claim. A store that does not mark leaves -the entry's `Expect` failing closed until the next pull. A whole-file flush or -unconditional commit rewrites every record under the writer's current schema -and keeps its `storedAt`, so another cache's `Expect` on a drifted record it -read before that rewrite fails until it re-reads the store. +Record identity for conditions is the record's version, the SHA-256 of its stored +bytes (cultcache-persistence-format.md). Any write that changes a record changes +its version, whatever storedAt it carries. - A failed condition is a lost race, not an error: `Commit` returns false and nothing is written, changed in memory, or published. `TryCommit` makes one @@ -268,17 +280,33 @@ read before that rewrite fails until it re-reads the store. match its content address, at most five times; a pull that has not settled by then throws. -**A plain flush and an unconditional commit are last-writer-wins: per file for -single-file stores, per key for directory stores.** Both run -under the same lock, so two writers never interleave bytes, but they compare -nothing. Only a conditional commit (`Expect` or `ExpectUnchanged`) protects -against another writer; processes sharing a store must all use it. +**In a store holding no variant, a plain flush and an unconditional commit are +last-writer-wins per staged key, in both store kinds.** Both run under the same lock, so two writers never +interleave bytes, but they compare nothing. Only a conditional commit +(`Expect` or `ExpectUnchanged`) tells a writer another writer moved the keys it +depends on. A writer wins only over a file it can read: a single-file store +refuses to replace a file whose header it cannot read, and leaves it as it was. -- A single-file store writes this cache's whole view of the file: a record - another writer added since this cache last pulled is gone. -- A directory store writes this cache's staged keys onto the current manifest: - another writer's unrelated keys survive, a key both wrote holds the last - write. +- A single-file store and a directory store apply this cache's staged writes and + removals onto what the store holds now: another writer's unrelated records + survive as stored, a key both wrote holds the last write. +- In both store kinds, a write or removal at a key whose durable record is under + an id the writer does not declare is refused with the typed schema conflict + before anything is written, and that record is reloaded as the store holds + it: dropped when foreign, held read-only when a type reads it + (cultcache-persistence-format.md, ownership by declaration). +- A removal of a key the cache does not hold travels as a check-only key: a + commit carries it in the request, a flush stages it, and either store kind + judges it under its lock with the keys it writes, never removing it. Every + writable store judges a batch's checks under its own lock: the stores the + batch does not commit through first (`JudgeChecks`), the home last. A batch + that only checks keys writes no file. `ExpectUnchanged` compares the foreign + records a store carries at the versions it read them; a store kind lists a + foreign record through `Foreign` with the version it read. +- A single-file store holding a variant (the directory store refuses variants) + is written only from what its writer last read. A record that moved refuses + the whole write with the typed write conflict, and the refused and moved keys + are reloaded (cultcache-persistence-format.md). - An unconditional commit writes exactly what a flush of the same staged state plus the batch would write. It also persists any single writes staged earlier in that store, and leaves the store clean. @@ -301,6 +329,25 @@ and Rust's open fails rather than waits while a C# writer holds the lock for `X` that `Doc` needs is declared in `Doc`'s assembly. - An assembly that declares nothing, and the store envelope, use the base options. +- A resolver may not claim a type a document's element lists pass through: the list's + element types, the types and members on the way to them, and the collection types + the walk passes (`List`, `T[]`, a collection class, a dictionary on the way). + Those are written by their `[Key]` layout so each element's id reaches disk, and the + codec refuses the document on first encode or decode, naming the resolver and the + type. The formatter `StandardResolver` or `StandardResolverAllowPrivate` hands out for a + type does not claim it: the extra non-public members the `AllowPrivate` formatter writes + are whole values the walk does not own. A type or member declaring + `[MessagePackFormatter]` is a whole value off the path, the supported way to give a type its + own formatter. A value list (`List`, a list of a non-keyed class) is not on the + path, so a resolver may claim it. +- The codec also checks, for every type on that path, that `StandardResolver` does not write it + against the walk. A type read by key may not be written by the generic collection resolver + (a `[MessagePackObject]` class that is also an `ICollection` or dictionary); it is + refused on first encode or decode, naming the type. Any other formatter, such as one + MessagePack's source generator emits, passes: the walk decides which types carry ids and + the door only verifies. It also refuses, typed, a type MessagePack cannot build a formatter + for (it throws, or `StandardResolver` returns none), and a resolver that throws while asked + about one. - Deserialization runs under untrusted-data security. `CultRecordRef` is a permitted dictionary key, compared by its key string. @@ -327,6 +374,37 @@ type name is the CLR full name, the same rule as `System.Int32`. A reader in another runtime needs only the component count and primitive type. CultMath has no serialization; a CultMath encoding, if added, uses this shape. +## Cacheless single-file access + +A single-file store file can be read and written without a cache, through public members of `SingleFileBackingStore` (for +example `SingleFileMessagePackBackingStore`): `ReadStoreSnapshot()`, `WriteRecord(record, catalogEntry, holdsIds, registry)`, +and `WriteDocument(key, document, storedAt, registry)`. Decoding a stored payload needs no file, so it is on the codec, not the store: +`CultCodec.DecodeDocument(key, payload)` (`CultCacheMessagePack.CreateCodec(registry)` makes the codec). + +- `ReadStoreSnapshot()` reads the file once, under the store's gate (or its cache's gate when attached, validated after it is taken), and returns the durable snapshot (records, schema catalog, + format stamp), or null when nothing is at the path. It throws `CultStoreUnreadableException` for a file the reader refuses. +- `WriteRecord` writes one record under the store's lock: the record replaces the one at its key, every other record and catalog + entry is copied forward as stored, and the file is replaced atomically. It takes a payload the caller already encoded, so a + payload it cannot read leaves the header as the file has it (`holdsIds` null). +- `WriteDocument` writes one typed document as a cache write does: it plans element ids, encodes the document as `T` (the type + its schema id and ids are taken from; a derived instance is stored as `T`) with the registry, and stores it as `WriteRecord` + does. A failed write gives the minted ids back. +- `CultCodec.DecodeDocument` decodes a payload read from the store as a cache load does: it mints the ids a load mints and + refuses what a cache load refuses. It reads no file and holds no state. + +The `registry` argument of `WriteRecord` and `WriteDocument` serves that one call: the store attaches it under its gate for the call +and keeps none, so the same call decides the same header on a fresh instance and on one that has served other calls. + +They share everything with a cache's store and nothing is re-implemented beside it: one reader (the stamp is judged before the +body, and a file the reader refuses is refused by a read and by a write alike), the store's lock, the atomic replace, and the +header rule (`holdsIds` true marks a v3 store; false leaves it as the rule decides; null, for a payload the caller cannot read, +keeps whatever the file already says). Ownership by declaration binds a cacheless write too: a record at the key under a schema id +the written catalog entry does not own or list as compatible is refused with `CultSchemaConflictException`, and the file is left as +it was. Element-id planning (`CultElementIds`) is internal to the library: a caller never plans or marks ids, the store does. A store +attached to a cache refuses `WriteRecord` and `WriteDocument` (write through the cache, whose staged writes land through the +same private write) with `InvalidOperationException`. CultMesh's single-file document helpers (`CultMesh.WriteSingleFileDocument`, +`ReadSingleFileDocument` and their payload forms) are built on these members and nothing else of the store. + ## Runtime status | Rule | C# | Rust | TypeScript | Python | diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index 4a4f7d37..ce276a0c 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -28,6 +28,83 @@ namespace GameCult.Caching public string[] CompatibleSchemaIds { get; set; } = Array.Empty(); public CultSchemaMemberCatalogEntry[] Members { get; set; } = Array.Empty(); + + // The catalog a write leaves, derived from the records the written store holds and nothing else. For each schema id a record + // carries, one entry that publishes it: for an id the write stages a record under, the entry the writer registered (or passed) + // for it, which owns the id; for every other id the arrived (durable) entry that owns it, else the first that lists it as + // compatible (the store that read it carries its bytes as read). A registered entry never publishes an id nothing staged sits under, so + // the caller passes registered entries only for the ids it writes. Entries are written as chosen, never merged; a registered + // descriptor wins over an arrived entry with the same own id. Two arrived entries with the same own id and different schema + // names, or a record no chosen entry publishes, refuse the write. + // Entries that tie are taken in one fixed order, so the catalog does not depend on the order they arrive in. + private static IEnumerable Canonical(IEnumerable entries) => entries + .OrderBy(entry => entry.SchemaName, StringComparer.Ordinal) + .ThenBy(entry => entry.ContentHash, StringComparer.Ordinal) + .ThenBy(entry => string.Join(",", entry.CompatibleSchemaIds), StringComparer.Ordinal); + + internal static CultSchemaCatalogEntry[] Derive( + IEnumerable records, + IReadOnlyCollection registered, + IReadOnlyCollection arrived) + { + var chosen = new Dictionary(StringComparer.Ordinal); + var carried = records.GroupBy(record => record.SchemaId, StringComparer.Ordinal).OrderBy(group => group.Key, StringComparer.Ordinal).ToArray(); + foreach (var group in carried) + { + var schemaId = group.Key; + var recordKey = group.Min(record => record.Key)!; + var ownRegisteredAll = Canonical(registered.Where(entry => string.Equals(entry.SchemaId, schemaId, StringComparison.Ordinal))).ToArray(); + var ownRegistered = ownRegisteredAll.FirstOrDefault(); + var ownArrived = Canonical(arrived.Where(entry => string.Equals(entry.SchemaId, schemaId, StringComparison.Ordinal))).ToArray(); + // Entries of one tier that share an own id must be one schema; a registered entry settles a disagreement among arrived ones. + var disagreeing = ownRegisteredAll.Length > 0 ? ownRegisteredAll : ownArrived; + if (disagreeing.Select(entry => entry.SchemaName).Distinct(StringComparer.Ordinal).Count() > 1) + { + var names = disagreeing.Select(entry => entry.SchemaName).Distinct(StringComparer.Ordinal).ToArray(); + throw new CultSchemaConflictException( + $"Schema id '{schemaId}' is claimed by schemas {string.Join(" and ", names.Select(name => $"'{name}'"))}; record '{recordKey}' cannot be written under it.", + schemaId, names, recordKey); + } + + CultSchemaCatalogEntry? pick; + bool isRegistered; + if (ownRegistered != null) + { + pick = ownRegistered; + isRegistered = true; + } + else if (ownArrived.Length > 0) + { + pick = ownArrived[0]; + isRegistered = false; + } + else + { + pick = Canonical(arrived.Where(entry => entry.CompatibleSchemaIds.Contains(schemaId, StringComparer.Ordinal))).FirstOrDefault(); + isRegistered = false; + } + + if (pick == null) + continue; + if (chosen.TryGetValue(pick.SchemaId, out var prior) && prior.Registered && !isRegistered) + continue; + chosen[pick.SchemaId] = (pick, isRegistered); + } + + foreach (var group in carried) + { + if (!chosen.Values.Any(pair => string.Equals(pair.Entry.SchemaId, group.Key, StringComparison.Ordinal) || + pair.Entry.CompatibleSchemaIds.Contains(group.Key, StringComparer.Ordinal))) + { + var recordKey = group.Min(record => record.Key)!; + throw new CultSchemaConflictException( + $"No schema entry publishes schema id '{group.Key}' of record '{recordKey}'.", + group.Key, chosen.Values.Select(pair => pair.Entry.SchemaName).ToArray(), recordKey); + } + } + + return chosen.Values.Select(pair => pair.Entry).ToArray(); + } } public sealed class CultSchemaMemberCatalogEntry @@ -60,30 +137,64 @@ namespace GameCult.Caching public CultVariantDelta? Variant { get; set; } } + // One record a write puts in a single-file store: encoded, with the catalog entry that publishes the id it is stored under and + // whether its document holds element ids (null when the writer cannot see, as for a raw payload). + internal sealed class CultStagedRecord + { + internal CultStagedRecord(CultPersistedRecord record, CultSchemaCatalogEntry entry, bool? holdsIds) + { + Record = record; + Entry = entry; + HoldsIds = holdsIds; + } + + internal CultPersistedRecord Record { get; } + internal CultSchemaCatalogEntry Entry { get; } + internal bool? HoldsIds { get; } + } + + /// + /// A record a store carries untouched because no registered type can read it, for example a record of a type this build does + /// not have, or one renamed under a new schema name without declaring its old id. The store keeps its bytes as the file holds + /// them, a write of another record keeps it, and a write that would replace or remove it is refused with a + /// . The cache never holds it. Declaring its schema id on a type claims it at the next load. + /// A record a type can read (through the catalog's schema name) but only by dropping persisted slots, and no type declares, is + /// not foreign: the cache holds it read-only, and the same refusal applies to a write or removal of it. A record a type reads + /// whole is owned and writable. + /// + public sealed class CultForeignRecord + { + internal CultForeignRecord(string key, string schemaId, string schemaName, string storedAt, string storedVersion) + { + Key = key; + SchemaId = schemaId; + SchemaName = schemaName; + StoredAt = storedAt; + StoredVersion = storedVersion; + } + + public string Key { get; } + + public string SchemaId { get; } + + // The name the store's catalog gives the schema. + public string SchemaName { get; } + + public string StoredAt { get; } + + // The version of the record's stored bytes when its store read it, in that store's spelling, given by the store through + // CacheBackingStore.Foreign. ExpectUnchanged compares it with the version on disk at the commit. + internal string StoredVersion { get; } + } + public sealed class CultPersistedStoreSnapshot { public const string FormatV1 = "cultcache.store.v1"; public const string FormatV2 = "cultcache.store.v2"; + public const string FormatV3 = "cultcache.store.v3"; public string FormatVersion { get; set; } = FormatV1; - /// - /// The N of a header shaped exactly cultcache.store.v<N> or cultcache.store.v<N>.<suffix>; - /// null for any other header. A header whose N exceeds what a runtime reads is a newer format, which that runtime - /// refuses and must never rewrite. - /// - internal static int? FormatNumber(string? header) - { - const string prefix = "cultcache.store.v"; - if (header == null || !header.StartsWith(prefix, StringComparison.Ordinal)) - return null; - var end = prefix.Length; - while (end < header.Length && header[end] >= '0' && header[end] <= '9') - end++; - if (end == prefix.Length || end - prefix.Length > 6 || (end < header.Length && header[end] != '.')) - return null; - return int.Parse(header.Substring(prefix.Length, end - prefix.Length), System.Globalization.CultureInfo.InvariantCulture); - } public CultSchemaCatalogEntry[] SchemaCatalog { get; set; } = Array.Empty(); public CultPersistedRecord[] Records { get; set; } = Array.Empty(); } @@ -181,6 +292,7 @@ namespace GameCult.Caching string schemaName, string schemaVersion, string schemaId, + IReadOnlyList compatibleSchemaIds, string contentHash, string canonicalSchemaJson, bool isGlobal, @@ -194,6 +306,7 @@ namespace GameCult.Caching SchemaName = schemaName; SchemaVersion = schemaVersion; SchemaId = schemaId; + CompatibleSchemaIds = compatibleSchemaIds; ContentHash = contentHash; CanonicalSchemaJson = canonicalSchemaJson; IsGlobal = isGlobal; @@ -219,6 +332,9 @@ namespace GameCult.Caching public string SchemaName { get; } public string SchemaVersion { get; } public string SchemaId { get; } + + /// The ids besides this type declared through . + public IReadOnlyList CompatibleSchemaIds { get; } public string ContentHash { get; } public string CanonicalSchemaJson { get; } public bool IsGlobal { get; } @@ -310,7 +426,7 @@ namespace GameCult.Caching SchemaVersion = SchemaVersion, ContentHash = ContentHash, CanonicalSchemaJson = CanonicalSchemaJson, - CompatibleSchemaIds = [SchemaId], + CompatibleSchemaIds = [SchemaId, .. CompatibleSchemaIds], Members = Members .OrderBy(member => member.Slot) .Select(member => new CultSchemaMemberCatalogEntry @@ -356,19 +472,13 @@ namespace GameCult.Caching string storedAt, CultDocumentDescriptor descriptor, object document) - : this(key, storedAt, descriptor, null, document, new PersistedIdentity(descriptor.SchemaId)) - { - } - - // A plain record its store read: persistedSchemaId is the schema id its durable bytes are stored under. - internal CultStoredDocument( - CultRecordKey key, - string storedAt, - CultDocumentDescriptor descriptor, - object document, - string persistedSchemaId) - : this(key, storedAt, descriptor, null, document, new PersistedIdentity(persistedSchemaId)) { + Key = key; + StoredAt = storedAt; + Descriptor = descriptor; + StoredSchemaId = descriptor.SchemaId; + StoredVersion = UnstoredVersion(); + _document = document; } // A variant: the delta is what is stored; the cache resolves it against its base in the hold that admits it. @@ -378,43 +488,47 @@ namespace GameCult.Caching CultDocumentDescriptor descriptor, CultVariantDelta variant, object? resolved, - string persistedSchemaId) - : this(key, storedAt, descriptor, variant, resolved, new PersistedIdentity(persistedSchemaId)) - { - } - - private CultStoredDocument( - CultRecordKey key, - string storedAt, - CultDocumentDescriptor descriptor, - CultVariantDelta? variant, - object? document, - PersistedIdentity persisted) + CultCodec? codec) { Key = key; StoredAt = storedAt; Descriptor = descriptor; + StoredSchemaId = descriptor.SchemaId; + StoredVersion = UnstoredVersion(); Variant = variant; - _document = document; - _persisted = persisted; + _document = resolved; + _codec = codec; } private readonly object? _document; - // One record's, shared by a variant and its resolved view: the store holds the one and the cache the other. - private readonly PersistedIdentity _persisted; + // What reads a variant's override values; null for a plain record. + private readonly CultCodec? _codec; public CultRecordKey Key { get; } - public string StoredAt { get; } + + // When the record was stored: metadata. A write that stores the record mints it a storedAt later than the one it replaces; + // no condition reads it. + public string StoredAt { get; internal set; } public CultDocumentDescriptor Descriptor { get; } - /// - /// The schema id this record is durably stored under, as its backing store last read or wrote it. It differs from - /// only for a record loaded under compatible schema drift that its - /// store has not rewritten since; a store records its rewrite of an entry it holds with . - /// Conditional commit compares this id and with the disk. Internal to the shipped stores. - /// - internal string PersistedSchemaId => _persisted.SchemaId; + // The schema id the record carries in its store: the id it was loaded under, which may be an older or foreign id that + // resolved to Descriptor, until a write of this record puts it under Descriptor's id (a write of an owned record moves it). + public string StoredSchemaId { get; internal set; } + + // True when the schema this record was loaded under left none of its persisted slots unread by Descriptor, so the type reads + // it whole and owns it (CultDocumentRegistry.Owns). Set by ToStoredDocument from the schema comparison's ignored slots, + // before any variant override is shed. A record this cache constructed or wrote is stored under Descriptor's own id, which + // the registry declares, so only a loaded record under another id is ever asked. The staging half asks it; + // ConditionsHold never reads it. Only the store's loaded view carries it: the cache's resolved copies and flattens are never + // asked. + internal bool StoredSchemaReadWhole { get; set; } + + // Names the bytes this record was read from or written as: the SHA-256 of its persisted encoding, set by the store that + // read or wrote them. A record no store has read or written has a version of its own that equals nothing else. + internal string StoredVersion { get; set; } + + private static string UnstoredVersion() => "unstored:" + Guid.NewGuid().ToString("N"); // The complete document. For a variant, the resolved view: derived, never written back. public object Document => _document ?? @@ -426,19 +540,98 @@ namespace GameCult.Caching // A plain record staged by Flatten: the one way a plain record may replace a variant at its key. internal bool Flattens { get; set; } - internal bool IsResolved => _document != null; + // True while this record holds element ids minted when it loaded, that no write has persisted (see MintElementIds). + internal bool IdsInMemoryOnly { get; set; } - internal CultStoredDocument Resolved(object document) => new(Key, StoredAt, Descriptor, Variant!, document, _persisted); - - // Reached only through CacheBackingStore.MarkWritten, the one way a store records a rewrite. - internal void MarkWritten() => _persisted.SchemaId = Descriptor.SchemaId; - - private sealed class PersistedIdentity + // True when some element this record persists holds a non-empty id, and the only thing a store reads to mark its header. + // It is read off the stored form each time it is asked, so a store asks it as it writes: a document changed after it was + // admitted, or loaded and changed in place, writes the header its content needs. A plain record persists its document; a + // variant persists its overrides, so it holds an id when an override value does (its base's ids are the base's). A + // variant built without a codec (a store loaded it with no cache attached) cannot see into its values and counts as holding + // one, so a store never writes an unmarked header over ids it could not see. + public bool HoldsIds { - public PersistedIdentity(string schemaId) => SchemaId = schemaId; - - public string SchemaId; + get + { + if (Variant == null) + return CultElementIds.Holds(Descriptor.DocumentType, _document); + if (_codec == null) + return true; + return Variant.Overrides.Any(entry => + { + var memberType = Descriptor.RichMembers.First(member => member.Slot == entry.Path[0].Slot).MemberType; + return CultElementIds.Holds(memberType, _codec.Deserialize(memberType, Descriptor.DocumentType, entry.Value)); + }); + } } + + internal CultStoredDocument Resolved(object document, CultVariantDelta delta, bool idsInMemoryOnly, CultCodec codec) => + new(Key, StoredAt, Descriptor, delta, document, codec) { IdsInMemoryOnly = idsInMemoryOnly, StoredSchemaId = StoredSchemaId, StoredVersion = StoredVersion }; + } + + // What the one claimant precedence (CultDocumentRegistry.Claimant) asks of the entries a reader holds: the entry owning an id, + // the entries listing an id compatible, the entries of a schema name, and how to name one in a refusal. + internal interface IClaimantLookup where T : class + { + T? Owner(string schemaId); + IReadOnlyList Listing(string schemaId); + IReadOnlyList Named(string schemaName); + ClaimantFacts Describe(T claimant); + } + + internal readonly struct ClaimantFacts + { + public ClaimantFacts(string schemaName, string schemaVersion, string typeLabel) + { + SchemaName = schemaName; + SchemaVersion = schemaVersion; + TypeLabel = typeLabel; + } + + public string SchemaName { get; } + public string SchemaVersion { get; } + public string TypeLabel { get; } + } + + // The lookup of a cacheless writer: the catalog entries the call writes, and the lent registry's other types. A written entry is + // the only description of its schema id, so the registry's type for that id is never asked: a registry adds claimants but never + // re-describes the written type. + internal sealed class CatalogEntryLookup : IClaimantLookup + { + private readonly IReadOnlyCollection _entries; + private readonly IClaimantLookup? _others; + + public CatalogEntryLookup(IReadOnlyCollection entries, CultDocumentRegistry? others) + { + _entries = entries; + _others = others?.Lookup; + } + + public CultSchemaCatalogEntry? Owner(string schemaId) => + _entries.FirstOrDefault(entry => string.Equals(entry.SchemaId, schemaId, StringComparison.Ordinal)) + ?? Other(_others?.Owner(schemaId)); + + public IReadOnlyList Listing(string schemaId) => + _entries.Where(entry => entry.CompatibleSchemaIds.Contains(schemaId, StringComparer.Ordinal)) + .Concat(Others(_others?.Listing(schemaId))) + .ToArray(); + + public IReadOnlyList Named(string schemaName) => + _entries.Where(entry => string.Equals(entry.SchemaName, schemaName, StringComparison.Ordinal)) + .Concat(Others(_others?.Named(schemaName))) + .ToArray(); + + public ClaimantFacts Describe(CultSchemaCatalogEntry claimant) => + new(claimant.SchemaName, claimant.SchemaVersion, claimant.SchemaId); + + // A registry type counts only when no written entry has its schema id. + private CultSchemaCatalogEntry? Other(CultDocumentDescriptor? descriptor) => + descriptor == null || _entries.Any(entry => string.Equals(entry.SchemaId, descriptor.SchemaId, StringComparison.Ordinal)) + ? null + : descriptor.ToCatalogEntry(); + + private IEnumerable Others(IEnumerable? descriptors) => + (descriptors ?? Array.Empty()).Select(Other).OfType(); } public sealed class CultDocumentRegistry @@ -546,52 +739,166 @@ namespace GameCult.Caching internal CultSchemaResolutionResult ResolvePersistedSchemaDetailed(string schemaId, IReadOnlyCollection catalog) { - var indexes = _indexes; - if (indexes.BySchemaId.TryGetValue(schemaId, out var exact)) + var local = Claimant(schemaId, catalog, out var persisted); + if (local == null) + throw new InvalidOperationException( + $"No local CultCache schema matches persisted schema '{persisted!.SchemaName}' ({schemaId})."); + if (persisted == null) { return new CultSchemaResolutionResult( - exact, + local, new CultSchemaMigrationReport { PersistedSchemaId = schemaId, - LocalSchemaId = exact.SchemaId, - PersistedSchemaName = exact.SchemaName, - LocalSchemaName = exact.SchemaName, + LocalSchemaId = local.SchemaId, + PersistedSchemaName = local.SchemaName, + LocalSchemaName = local.SchemaName, Kind = CultSchemaMigrationKind.Exact }); } - var persisted = catalog.FirstOrDefault(entry => string.Equals(entry.SchemaId, schemaId, StringComparison.Ordinal)); + return BuildCompatibleResolutionResult(persisted, local); + } + + // The registered type a record under this schema id belongs to, or null when no registered type claims it: a foreign record, + // which a store carries untouched. Deciding this never decodes the record or compares shapes, so a store can classify what a + // file holds without reading a payload. persisted is the catalog entry the record resolves through, and null when a + // registered type owns the id. Claimants that cannot be told apart refuse. Branches past the listers (a local id the persisted + // entry lists, then the schema name) pick a type to read with. They confer no ownership: Owns decides what a writer may write. + internal CultDocumentDescriptor? Claimant( + string schemaId, + IReadOnlyCollection catalog, + out CultSchemaCatalogEntry? persisted) => + Claimant(schemaId, catalog, _indexes, out persisted); + + // The only claimant precedence: which entry reads a persisted catalog entry, over a lookup of the entries a reader holds (a + // registry's types, or the entries a cacheless writer writes). Exact id, then the one entry listing the id, then a local id the + // persisted entry lists, then the schema name. Every caller, cache or cacheless, asks it; none writes a precedence of its own. + internal static T? Claimant( + string schemaId, + IReadOnlyCollection catalog, + IClaimantLookup lookup, + out CultSchemaCatalogEntry? persisted) + where T : class + { + persisted = null; + if (lookup.Owner(schemaId) is { } exact) + return exact; + + persisted = PublishedBy(schemaId, catalog); if (persisted == null) { throw new InvalidOperationException($"Persisted schema '{schemaId}' is not present in the embedded catalog."); } - foreach (var compatibleSchemaId in persisted.CompatibleSchemaIds.Where(candidate => !string.IsNullOrWhiteSpace(candidate))) + // No type owns the id, so the one type that lists it is the record's type. Two listers and no owner cannot say which. + var listers = lookup.Listing(schemaId); + if (listers.Count > 0) { - if (indexes.BySchemaId.TryGetValue(compatibleSchemaId, out var compatibleLocal)) - { - return BuildCompatibleResolutionResult(persisted, compatibleLocal); - } + if (listers.Count > 1) + throw new CultSchemaConflictException( + $"CultCache schema id '{schemaId}' is owned by no registered type and declared compatible by several " + + $"({string.Join(", ", listers.Select(lister => lookup.Describe(lister).TypeLabel))}), so a record under it resolves to none. " + + "Register the type that owns the id, or declare it on one type.", + schemaId, + listers.Select(lister => lookup.Describe(lister).SchemaName).ToArray(), + string.Empty); + return listers[0]; } - if (indexes.BySchemaName.TryGetValue(persisted.SchemaName, out var localVersions)) + foreach (var compatibleSchemaId in persisted.CompatibleSchemaIds.Where(candidate => !string.IsNullOrWhiteSpace(candidate))) { - if (localVersions.Length == 1) - { - return BuildCompatibleResolutionResult(persisted, localVersions[0]); - } + if (lookup.Owner(compatibleSchemaId) is { } compatibleLocal) + return compatibleLocal; + } + + var localVersions = lookup.Named(persisted.SchemaName); + if (localVersions.Count > 0) + { + if (localVersions.Count == 1) + return localVersions[0]; throw new InvalidOperationException( $"CultCache schema name '{persisted.SchemaName}' is ambiguous across local versions " + - $"[{string.Join(", ", localVersions.Select(candidate => candidate.SchemaVersion).OrderBy(version => version, StringComparer.Ordinal))}]. " + + $"[{string.Join(", ", localVersions.Select(candidate => lookup.Describe(candidate).SchemaVersion).OrderBy(version => version, StringComparer.Ordinal))}]. " + "Persisted schema compatibility must identify an explicit schema id."); } - throw new InvalidOperationException( - $"No local CultCache schema matches persisted schema '{persisted.SchemaName}' ({schemaId})."); + return null; } + // The catalog entry a record under this id is published by: the entry that owns the id, else the first that lists it compatible. + internal static CultSchemaCatalogEntry? PublishedBy(string schemaId, IReadOnlyCollection catalog) => + catalog.FirstOrDefault(entry => string.Equals(entry.SchemaId, schemaId, StringComparison.Ordinal)) + ?? catalog.FirstOrDefault(entry => entry.CompatibleSchemaIds.Contains(schemaId, StringComparer.Ordinal)); + + // The one answer to whether a writer may write, relabel or remove a record stored under this id + // (variants:ruling:drift-record-write-ownership): a registered type declares the id (owns it or lists it compatible), or the + // type that reads it reads it whole. Read live, so a type registered later that declares the id claims its records at their + // next write. A record read only by dropping slots is not owned: declaring its id compatible is the consent to the loss. + // Header only: the persisted catalog entry that publishes the id is compared with the claimant's entry, never a payload. A + // claimant that cannot be named (ambiguous, id missing from the catalog, shapes incompatible) owns nothing. + internal bool Owns(string schemaId, IReadOnlyCollection catalog) => + Owns(schemaId, catalog, Declared, _indexes, descriptor => descriptor.ToCatalogEntry(), _ => true); + + // The registry's lookup, for a cacheless writer that lends it only its other types. + internal IClaimantLookup Lookup => _indexes; + + // The rule itself, for any writer: declared, or the claimant (picked by the one Claimant over a lookup) reads the record's + // published entry whole. The lookup is a registry's indexes, or the entries a registry-less cacheless writer writes. mayClaim + // is a condition on the claimant's answer, not a second lookup: a cache accepts any claimant, a cacheless writer only one of + // the entries the call writes, so it never owns a record another registered type reads. + internal static bool Owns( + string schemaId, + IReadOnlyCollection catalog, + Func declared, + IClaimantLookup lookup, + Func entryOf, + Func mayClaim) + where T : class + { + if (declared(schemaId)) + return true; + try + { + var claimant = Claimant(schemaId, catalog, lookup, out var persisted); + if (persisted == null || claimant == null) + return false; + var claimantEntry = entryOf(claimant); + return mayClaim(claimantEntry) && ReadsWhole(persisted, claimantEntry); + } + catch (InvalidOperationException) + { + return false; + } + } + + // Owns' declared half: a registered type owns the id or lists it compatible. + internal bool Declared(string schemaId) + { + var indexes = _indexes; + return indexes.BySchemaId.ContainsKey(schemaId) || indexes.ByCompatibleSchemaId.ContainsKey(schemaId); + } + + // A persisted entry a local entry reads whole: identical content, or compatible shapes that leave no persisted slot ignored. + internal static bool ReadsWhole(CultSchemaCatalogEntry persisted, CultSchemaCatalogEntry local) => + string.Equals(persisted.ContentHash, local.ContentHash, StringComparison.Ordinal) || + CompareSchemaShapes(ShapeOf(persisted), ShapeOf(local)) is { IsCompatible: true, IgnoredExtraSlots.Length: 0 }; + + private static CanonicalSchemaShape ShapeOf(CultSchemaCatalogEntry entry) => + new( + entry.SchemaName, + entry.SchemaVersion, + entry.Members.Select(member => new CanonicalSchemaMember( + member.Slot, + member.MemberName, + member.TypeName, + member.IsReference, + member.IsMany, + member.TargetSchemaName, + member.IsName, + member.IndexAlias)).ToArray()); + private CultDocumentDescriptor RegisterDescriptor(CultDocumentDescriptor descriptor) { lock (_registrationGate) @@ -614,63 +921,46 @@ namespace GameCult.Caching if (indexes.ByType.TryGetValue(descriptor.DocumentType, out var registered)) return registered; + // One type owns a schema, whatever its declaration. Its id hashes the schema's name, version and members, so a second + // type of the same name and version is refused here whether or not it hashes alike. Other types may list the id as + // compatible beside its owner, as a v2 class does that keeps v1's class registered: a record under the id resolves to + // its owner, and to a lister only when nothing owns it. Neither depends on the order types were registered in. indexes.BySchemaName.TryGetValue(descriptor.SchemaName, out var schemaNameVersions); var schemaVersionOwner = schemaNameVersions?.FirstOrDefault(candidate => string.Equals(candidate.SchemaVersion, descriptor.SchemaVersion, StringComparison.Ordinal)); if (schemaVersionOwner != null) - { - if (IsExactWireAlias(schemaVersionOwner, descriptor)) - { - indexes.ByType[descriptor.DocumentType] = descriptor; - return descriptor; - } - - throw DuplicateSchemaRegistration( - $"schema name '{descriptor.SchemaName}' version '{descriptor.SchemaVersion}'", - schemaVersionOwner.DocumentType, - descriptor.DocumentType); - } - - if (indexes.BySchemaId.TryGetValue(descriptor.SchemaId, out var schemaIdOwner)) - { - throw DuplicateSchemaRegistration( - $"schema id '{descriptor.SchemaId}'", - schemaIdOwner.DocumentType, - descriptor.DocumentType); - } + throw SchemaClaimedTwice(schemaVersionOwner, descriptor); indexes.ByType[descriptor.DocumentType] = descriptor; indexes.BySchemaId[descriptor.SchemaId] = descriptor; + foreach (var compatibleSchemaId in descriptor.CompatibleSchemaIds) + indexes.ByCompatibleSchemaId[compatibleSchemaId] = indexes.ByCompatibleSchemaId.TryGetValue(compatibleSchemaId, out var listers) + ? [.. listers, descriptor] + : [descriptor]; indexes.BySchemaName[descriptor.SchemaName] = schemaNameVersions == null ? [descriptor] : [.. schemaNameVersions, descriptor]; return descriptor; } - private static bool IsExactWireAlias( - CultDocumentDescriptor canonical, - CultDocumentDescriptor candidate) => - string.Equals(canonical.SchemaName, candidate.SchemaName, StringComparison.Ordinal) && - string.Equals(canonical.SchemaVersion, candidate.SchemaVersion, StringComparison.Ordinal) && - string.Equals(canonical.SchemaId, candidate.SchemaId, StringComparison.Ordinal) && - string.Equals(canonical.ContentHash, candidate.ContentHash, StringComparison.Ordinal) && - string.Equals(canonical.CanonicalSchemaJson, candidate.CanonicalSchemaJson, StringComparison.Ordinal); - - private static InvalidOperationException DuplicateSchemaRegistration( - string identity, - Type existingType, - Type claimedType) => + private static CultSchemaConflictException SchemaClaimedTwice( + CultDocumentDescriptor existing, + CultDocumentDescriptor claimed) => new( - $"CultCache {identity} is already registered to CLR type " + - $"'{existingType.FullName}' and cannot also be claimed by '{claimedType.FullName}'. " + - "Schema compatibility must be declared through explicit persisted-schema alias metadata."); + $"CultCache schema '{claimed.SchemaName}' version '{claimed.SchemaVersion}' is already registered to CLR type " + + $"'{existing.DocumentType.FullName}' and cannot also be claimed by '{claimed.DocumentType.FullName}'. " + + "A schema is owned by one registered type; others may only declare its id compatible.", + claimed.SchemaId, + new[] { existing.SchemaName, claimed.SchemaName }, + string.Empty); - private sealed class RegistryIndexes + private sealed class RegistryIndexes : IClaimantLookup { public RegistryIndexes() { ByType = new Dictionary(); BySchemaId = new Dictionary(StringComparer.Ordinal); + ByCompatibleSchemaId = new Dictionary(StringComparer.Ordinal); BySchemaName = new Dictionary(StringComparer.Ordinal); } @@ -678,12 +968,26 @@ namespace GameCult.Caching { ByType = new Dictionary(source.ByType); BySchemaId = new Dictionary(source.BySchemaId, StringComparer.Ordinal); + ByCompatibleSchemaId = new Dictionary(source.ByCompatibleSchemaId, StringComparer.Ordinal); BySchemaName = new Dictionary(source.BySchemaName, StringComparer.Ordinal); } public Dictionary ByType { get; } public Dictionary BySchemaId { get; } + // Every type declaring the id compatible, in registration order; resolution never depends on that order. + public Dictionary ByCompatibleSchemaId { get; } public Dictionary BySchemaName { get; } + + public CultDocumentDescriptor? Owner(string schemaId) => BySchemaId.GetValueOrDefault(schemaId); + + public IReadOnlyList Listing(string schemaId) => + ByCompatibleSchemaId.GetValueOrDefault(schemaId) ?? Array.Empty(); + + public IReadOnlyList Named(string schemaName) => + BySchemaName.GetValueOrDefault(schemaName) ?? Array.Empty(); + + public ClaimantFacts Describe(CultDocumentDescriptor claimant) => + new(claimant.SchemaName, claimant.SchemaVersion, claimant.DocumentType.FullName ?? string.Empty); } private static CultDocumentDescriptor BuildDescriptor(Type type) @@ -692,6 +996,9 @@ namespace GameCult.Caching ?? throw new InvalidOperationException( $"Type {type.FullName} is not marked with {nameof(CultDocumentAttribute)}."); var members = DiscoverMembers(type); + var ids = CultElementIds.Inspect(type); + if (ids.Problems.Count > 0) + throw new InvalidOperationException(ids.Problems[0]); var nameMember = members.FirstOrDefault(member => member.IsName); var indexAccessors = members .Where(member => member.IndexAlias != null) @@ -716,12 +1023,17 @@ namespace GameCult.Caching var contentHash = Sha256(schemaJson); var semanticFingerprint = BuildSemanticFingerprint(attribute.SchemaName, attribute.SchemaVersion, descriptorMembers); var schemaId = Sha256(semanticFingerprint); + var compatibleSchemaIds = attribute.CompatibleSchemaIds + .Where(id => !string.IsNullOrWhiteSpace(id) && !string.Equals(id, schemaId, StringComparison.Ordinal)) + .Distinct(StringComparer.Ordinal) + .ToArray(); return new CultDocumentDescriptor( type, attribute.SchemaName, attribute.SchemaVersion, schemaId, + compatibleSchemaIds, contentHash, schemaJson, type.GetCustomAttribute() != null, @@ -836,31 +1148,7 @@ namespace GameCult.Caching }); } - var persistedSchema = new CanonicalSchemaShape( - persisted.SchemaName, - persisted.SchemaVersion, - persisted.Members.Select(member => new CanonicalSchemaMember( - member.Slot, - member.MemberName, - member.TypeName, - member.IsReference, - member.IsMany, - member.TargetSchemaName, - member.IsName, - member.IndexAlias)).ToArray()); - var localSchema = new CanonicalSchemaShape( - local.SchemaName, - local.SchemaVersion, - local.Members.Select(member => new CanonicalSchemaMember( - member.Slot, - member.MemberName, - member.TypeName, - member.IsReference, - member.IsMany, - member.TargetSchemaName, - member.IsName, - member.IndexAlias)).ToArray()); - var comparison = CompareSchemaShapes(persistedSchema, localSchema); + var comparison = CompareSchemaShapes(ShapeOf(persisted), ShapeOf(local.ToCatalogEntry())); if (!comparison.IsCompatible) { throw new InvalidOperationException( @@ -910,11 +1198,6 @@ namespace GameCult.Caching var ignoredExtraSlots = new List(); var defaultedMissingSlots = new List(); - if (!string.Equals(persisted.SchemaName, local.SchemaName, StringComparison.Ordinal)) - { - errors.Add($"schema name drift '{persisted.SchemaName}' -> '{local.SchemaName}'"); - } - var persistedBySlot = persisted.Members.ToDictionary(member => member.Slot); var localBySlot = local.Members.ToDictionary(member => member.Slot); @@ -1732,46 +2015,60 @@ namespace GameCult.Caching internal CultCommitRequest( IReadOnlyList upserts, IReadOnlyList deletes, - IReadOnlyList<(CultRecordKey Key, string? SchemaId, string? StoredAt)> expected, + IReadOnlyList checks, + IReadOnlyList<(CultRecordKey Key, string? Version)> expected, bool expectUnchanged) { Upserts = upserts; Deletes = deletes; + Checks = checks; Expected = expected; ExpectUnchanged = expectUnchanged; } public IReadOnlyList Upserts { get; } public IReadOnlyList Deletes { get; } - public IReadOnlyList<(CultRecordKey Key, string? SchemaId, string? StoredAt)> Expected { get; } + + /// + /// Keys the batch removes that its cache does not hold. Such a removal removes nothing, so it is never in + /// ; but a store that holds a record at one of these keys under an id no registered type declares + /// refuses the commit, as it refuses a write of that record. The home's judges + /// these keys under its lock, with the keys it writes; every other attached store judges them first through + /// , each under its own lock, and a refusal anywhere writes nothing. A judged + /// key is never removed: a record a declared type reads stays where it is. + /// + public IReadOnlyList Checks { get; } + public IReadOnlyList<(CultRecordKey Key, string? Version)> Expected { get; } public bool ExpectUnchanged { get; } public bool HasConditions => Expected.Count > 0 || ExpectUnchanged; - // durable is what the store holds on disk now; observed is what its cache last loaded or committed. - // Identity is (the schema id the record is persisted under, storedAt). A value change mints a later storedAt; a - // whole-file write keeps storedAt and rewrites a drifted record under another schema id, which the schema half sees. - public bool ConditionsHold(IReadOnlyCollection durable, IEnumerable observed) + /// + /// Whether the batch's conditions hold. is the version of each record the store holds on disk now, + /// by key; is what the store's cache last loaded or committed, and the + /// foreign records the store carries as it last read them (; empty for a home + /// with none). Under each foreign record compares against the version it had when the store + /// read it, as a record the cache holds compares against its own. Identity is the record's version, the SHA-256 of its stored + /// bytes, which names its schema id and storedAt too. + /// + public bool ConditionsHold(IReadOnlyDictionary durable, IEnumerable observed, IEnumerable foreign) { - var byKey = durable.ToDictionary(record => record.Key, StringComparer.Ordinal); - foreach (var (key, schemaId, storedAt) in Expected) + foreach (var (key, version) in Expected) { - byKey.TryGetValue(key.Value, out var record); - var holds = schemaId == null - ? record == null - : record != null && record.SchemaId == schemaId && record.StoredAt == storedAt; + var holds = durable.TryGetValue(key.Value, out var stored) + ? version != null && string.Equals(stored, version, StringComparison.Ordinal) + : version == null; if (!holds) return false; } return !ExpectUnchanged || - durable.Select(record => Identity(record.Key, record.SchemaId, record.StoredAt)) - .OrderBy(identity => identity, StringComparer.Ordinal) + durable.Select(pair => (pair.Key, Version: pair.Value)) + .OrderBy(pair => pair.Key, StringComparer.Ordinal) .SequenceEqual(observed - .Select(stored => Identity(stored.Key.Value, stored.PersistedSchemaId, stored.StoredAt)) - .OrderBy(identity => identity, StringComparer.Ordinal)); + .Select(stored => (Key: stored.Key.Value, Version: stored.StoredVersion)) + .Concat(foreign.Select(record => (record.Key, Version: record.StoredVersion))) + .OrderBy(pair => pair.Key, StringComparer.Ordinal)); } - - private static string Identity(string key, string schemaId, string storedAt) => $"{key}\n{schemaId}\n{storedAt}"; } // One staged write: a plain record (Stored), a variant (Variant), a flatten, or, with none of those, a removal. @@ -1797,7 +2094,7 @@ namespace GameCult.Caching { private readonly CultCache _cache; internal readonly Dictionary Operations = new(StringComparer.Ordinal); - internal readonly List<(CultRecordKey Key, string? SchemaId, string? StoredAt)> Expected = new(); + internal readonly List<(CultRecordKey Key, string? Version)> Expected = new(); internal bool ExpectsUnchanged; internal bool Sealed; @@ -1844,8 +2141,8 @@ namespace GameCult.Caching public void Expect(CultRecordKey key, object? current) { ThrowIfSealed(); - var identity = _cache.Observe(key, current); - Expected.Add((key, identity?.SchemaId, identity?.StoredAt)); + var observed = _cache.Observe(key, current); + Expected.Add((key, observed?.StoredVersion)); } public void ExpectUnchanged() @@ -1891,6 +2188,10 @@ namespace GameCult.Caching private long _sequence; // What this thread's outermost hold admitted, published by that hold when it exits. [ThreadStatic] private static Hold? _held; + // The store gate this thread holds (InStoreHold), if any. With _held this is the whole of what a thread holds: at most one + // cache's gate and at most one store's. A store's gate may be taken inside a cache's, never the reverse, so while this is set a + // thread enters no cache gate it does not already hold, reads included (Gate). + [ThreadStatic] private static object? _storeHeld; // Journals run under the gate, in registration order; both fields are read and written only under it. private readonly List>>> _journals = new(); private bool _journaling; @@ -1906,8 +2207,24 @@ namespace GameCult.Caching public CultDocumentRegistry Registry => _registry; - // Read under the gate, by a store judging a merge. - internal bool HoldsVariants => _variantKeys.Count > 0; + // A store wrote: every record it wrote is stored under its type's own id at the storedAt the write gave it, with the + // version of the bytes it wrote, and a plain record carries the ids it held, so it is no longer in-memory only. A variant's ids are not cleared here: the + // store wrote the delta it was handed, and only a write of the variant changes that. + internal void Wrote(IEnumerable<(CultPersistedRecord Record, string Version)> written) => Held(() => + { + foreach (var (record, version) in written) + { + if (!_entries.TryGetValue(record.Key, out var entry)) + continue; + entry.StoredSchemaId = record.SchemaId; + entry.StoredAt = record.StoredAt; + entry.StoredVersion = version; + if (entry.Variant == null) + entry.IdsInMemoryOnly = false; + } + + return true; + }); public CultCodec? Codec => _codec; @@ -1924,11 +2241,39 @@ namespace GameCult.Caching public CultVariantOverride Override(string member, object? value) where T : class => Override(typeof(T), member, value); + // The one-shot rewrite for stores written before element ids existed: upserts every record whose ids exist only in + // memory (minted when it loaded), through the normal write path, one commit per home store. Returns how many records + // it rewrote; a second call finds none. A selected record that is neither declared nor read whole is read-only, + // so its home store's commit is refused with CultSchemaConflictException; commits to earlier stores stand. + public int MintElementIds() + { + var groups = Held(() => _entries.Values + .Where(entry => entry.IdsInMemoryOnly) + .GroupBy(entry => Home(entry.Descriptor.DocumentType)) + .Select(group => group.ToArray()) + .ToArray()); + foreach (var group in groups) + { + Commit(batch => + { + foreach (var entry in group) + { + if (entry.Variant is { } delta) + batch.UpsertVariant(entry.Key, new CultRecordKey(delta.BaseKey), delta.Overrides); + else + batch.Upsert(entry.Descriptor.DocumentType, entry.Document, entry.Key); + } + }); + } + + return groups.Sum(group => group.Length); + } + public bool IsDirty { get { - lock (_gate) + lock (Gate()) return _stores.Count == 0 ? _dirtyInMemory : _stores.Any(entry => entry.Store.IsDirty); } } @@ -1941,7 +2286,7 @@ namespace GameCult.Caching { get { - lock (_gate) + lock (Gate()) return _stores.Select(entry => entry.Store).ToArray(); } } @@ -1950,7 +2295,7 @@ namespace GameCult.Caching { get { - lock (_gate) + lock (Gate()) return _entries.Values.Select(entry => entry.Document).ToArray(); } } @@ -1959,7 +2304,7 @@ namespace GameCult.Caching { get { - lock (_gate) + lock (Gate()) return _entries.Values .OrderBy(entry => entry.Descriptor.SchemaName, StringComparer.Ordinal) .ThenBy(entry => entry.Key.Value, StringComparer.Ordinal) @@ -2048,9 +2393,14 @@ namespace GameCult.Caching { if (store == null) throw new ArgumentNullException(nameof(store)); homes ??= Array.Empty(); - Held(() => + // The only writer of store.Cache, holding the cache's gate and the store's. It publishes Cache before the pull, so the store is + // attached for the whole of its hold and every entry point agrees: a journal the pull runs that calls the store is a journal + // entering its cache and is refused, as it is after the attach; every other thread waits on the cache's or the store's gate. + // A failed pull takes Cache back in that same hold. Another thread's unlocked guess may read the tentative cache, but none runs a + // body against it: the guess is checked under that cache's gate, which the attach holds, and fails once the pull retracts it. + Held(() => store.UnderStoreGate(() => { - if (store.Loaded != null) + if (store.Cache != null) throw new InvalidOperationException($"Backing store {store} is already attached to a cache."); if (store.IsDirty) throw new InvalidOperationException($"Backing store {store} has staged writes; attach it clean."); @@ -2077,10 +2427,9 @@ namespace GameCult.Caching "attach routed stores before the untyped store."); } - store.AttachRegistry(_registry); + store.UseRegistry(_registry); store.Cache = this; store.Loaded = (loaded, dropped) => Admit(loaded, dropped, store, (_, _) => CultCommitOutcome.Committed); - store.Judging = (arriving, departing) => { Resolve(arriving, departing, store); }; _stores.Add(candidate); try { @@ -2090,12 +2439,12 @@ namespace GameCult.Caching { _stores.Remove(candidate); store.Loaded = null; - store.Judging = null; + store.UseRegistry(null); store.Cache = null; throw; } return true; - }); + })); } public Task PullAllBackingStoresAsync() @@ -2126,7 +2475,7 @@ namespace GameCult.Caching { foreach (var (store, _) in _stores) { - if (!store.IsReadOnly && store.IsDirty) + if (!store.IsReadOnly && (store.IsDirty || store.HasStagedChecks)) store.PushAll(); } return true; @@ -2143,12 +2492,14 @@ namespace GameCult.Caching return UpsertAsync(document, handle); } + // Refused with CultSchemaConflictException when the key holds a record stored under an id no registered type declares. public Task> UpsertAsync(T document, CultRecordHandle? handle = null) { if (document == null) throw new ArgumentNullException(nameof(document)); return Task.FromResult(new CultRecordHandle(Write(document, handle?.Key))); } + // Refused with CultSchemaConflictException when the key holds a record stored under an id no registered type declares. public Task UpsertAsync(Type documentType, object document, CultRecordKey? key = null) { RequireInstanceOf(documentType, document); @@ -2156,6 +2507,7 @@ namespace GameCult.Caching } // A single-call variant write; a batch stages the same thing with CultCacheBatch.UpsertVariant. + // Refused with CultSchemaConflictException when the key holds a record stored under an id no registered type declares. public Task UpsertVariantAsync(CultRecordKey key, CultRecordKey baseKey, IEnumerable? overrides = null) { var delta = new CultVariantDelta(baseKey.Value, (overrides ?? Array.Empty()).ToArray()); @@ -2172,6 +2524,7 @@ namespace GameCult.Caching } // Replaces a variant with a plain record holding its resolved document (Q1a's explicit operation). + // Refused with CultSchemaConflictException when the key holds a record stored under an id no registered type declares. public Task FlattenAsync(CultRecordKey key) => Task.FromResult(Held(() => { var stored = StampFlatten(key); @@ -2183,10 +2536,22 @@ namespace GameCult.Caching return key; })); + /// + /// Removes the record at ; false when the cache holds nothing there. + /// + /// + /// A record stored under an id no registered type declares, read-only or foreign, is never removed: removing one the cache + /// holds is refused with now. A key the cache does not hold costs nothing here and + /// returns false; each writable store judges it at its next flush or commit, under its lock, and refuses that flush or + /// commit if it holds a record there under such an id (a read-only store answers now, from what it last loaded). + /// public bool Remove(CultRecordKey key) => Held(() => { if (!_entries.TryGetValue(key.Value, out var existing)) + { + RefuseRemovalOfUndeclared(new[] { key.Value }); return false; + } Admit(Array.Empty(), new[] { existing }, null, (home, _) => { home?.Delete(existing); @@ -2222,20 +2587,20 @@ namespace GameCult.Caching public CultRecordHandle? TryGetHandle(T document) { if (document == null) throw new ArgumentNullException(nameof(document)); - lock (_gate) + lock (Gate()) return _handles.TryGetValue(document, out var box) ? new CultRecordHandle(box.Key) : null; } // The stored form of a record: for a variant, the delta beside the resolved document. public CultStoredDocument? GetStored(CultRecordKey key) { - lock (_gate) + lock (Gate()) return _entries.TryGetValue(key.Value, out var stored) ? stored : null; } public object? Get(CultRecordKey key) { - lock (_gate) + lock (Gate()) return _entries.TryGetValue(key.Value, out var stored) ? stored.Document : null; } @@ -2247,7 +2612,7 @@ namespace GameCult.Caching // Every change whose Sequence is at or below the returned one is reflected in the returned document. public (object? Document, long Sequence) GetWithSequence(CultRecordKey key) { - lock (_gate) + lock (Gate()) return (_entries.TryGetValue(key.Value, out var stored) ? stored.Document : null, _sequence); } @@ -2259,7 +2624,7 @@ namespace GameCult.Caching public IEnumerable GetAll() where T : class { - lock (_gate) + lock (Gate()) return _entries.Values.Select(entry => entry.Document).OfType().ToArray(); } @@ -2293,7 +2658,7 @@ namespace GameCult.Caching { Held(() => { - if (FlushAttachedStoresOnDispose && IsDirty) + if (FlushAttachedStoresOnDispose && (IsDirty || _stores.Any(entry => entry.Store.HasStagedChecks))) FlushAllBackingStores(); return true; }); @@ -2340,7 +2705,7 @@ namespace GameCult.Caching internal CultStoredDocument CreateStoredDocument(object document, CultRecordKey? key) { var descriptor = _registry.GetRequired(document.GetType()); - lock (_gate) + lock (Gate()) return Stamp( key ?? (_handles.TryGetValue(document, out var box) ? box.Key : descriptor.IsGlobal ? new CultRecordKey($"global:{descriptor.SchemaId}") @@ -2351,18 +2716,18 @@ namespace GameCult.Caching // Call under the gate that admits the result, so no load can land a later storedAt in between. private CultStoredDocument Stamp(CultRecordKey key, CultDocumentDescriptor descriptor, object document) => - new(key, MintStoredAt(PreviousStoredAt(key)), descriptor, document); + new(key, MintStoredAt(PreviousStoredAt(key), Clock()), descriptor, document); // An unresolved variant: the admitting hold resolves it against its base before anything lands. private CultStoredDocument StampVariant(CultRecordKey key, CultDocumentDescriptor descriptor, CultVariantDelta delta) => - new(key, MintStoredAt(PreviousStoredAt(key)), descriptor, delta, null, descriptor.SchemaId); + new(key, MintStoredAt(PreviousStoredAt(key), Clock()), descriptor, delta, null, _codec); // The plain record a Flatten stages: the variant's resolved document, under the variant's own key and schema. private CultStoredDocument StampFlatten(CultRecordKey key) { if (!_entries.TryGetValue(key.Value, out var existing) || existing.Variant == null) throw new InvalidOperationException($"Flatten({key.Value}): the record is not a variant."); - return new CultStoredDocument(key, MintStoredAt(existing.StoredAt), existing.Descriptor, existing.Document) { Flattens = true }; + return new CultStoredDocument(key, MintStoredAt(existing.StoredAt, Clock()), existing.Descriptor, existing.Document) { Flattens = true }; } private string? PreviousStoredAt(CultRecordKey key) => _entries.TryGetValue(key.Value, out var previous) ? previous.StoredAt : null; @@ -2399,16 +2764,15 @@ namespace GameCult.Caching } } - // The durable identity of the record holding current, read under the gate, where a store's write moves it. - internal (string SchemaId, string StoredAt)? Observe(CultRecordKey key, object? current) + internal CultStoredDocument? Observe(CultRecordKey key, object? current) { if (current == null) return null; - lock (_gate) + lock (Gate()) { if (_handles.TryGetValue(current, out var box) && box.Key.Equals(key) && _entries.TryGetValue(key.Value, out var stored) && ReferenceEquals(stored.Document, current)) - return (stored.PersistedSchemaId, stored.StoredAt); + return stored; } throw new ArgumentException($"Expect({key.Value}) was given an instance this cache does not hold at that key.", nameof(current)); @@ -2417,9 +2781,8 @@ namespace GameCult.Caching // The wall clock behind every storedAt. Internal: a test that writes a golden store pins it, so the bytes reproduce. internal Func Clock { get; set; } = () => DateTimeOffset.UtcNow; - private string MintStoredAt(string? previous) + internal static string MintStoredAt(string? previous, DateTimeOffset now) { - var now = Clock(); if (previous != null && DateTimeOffset.TryParse(previous, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind, out var last) && now <= last) @@ -2461,14 +2824,33 @@ namespace GameCult.Caching : op.Variant != null ? StampVariant(op.Key, DescriptorOfBase(op.Key, op.Variant.BaseKey, batch), op.Variant) : StampFlatten(op.Key)) .ToArray(); + // A removal of a key the cache does not hold removes nothing: it is a check every store the commit reaches judges under its lock. + var checks = batch.Operations.Values + .Where(op => op.IsRemoval && !_entries.ContainsKey(op.Key.Value)) + .Select(op => op.Key.Value) + .ToArray(); var deletes = batch.Operations.Values .Where(op => op.IsRemoval && _entries.ContainsKey(op.Key.Value)) .Select(op => _entries[op.Key.Value]) .ToArray(); return Admit(upserts, deletes, null, (home, resolved) => { + // A batch that only checks keys names no home; it commits through the cache's one writable store, if it has one. + if (home == null && checks.Length > 0 && _stores.Where(entry => !entry.Store.IsReadOnly).Take(2).ToArray() is { Length: 1 } only) + home = only[0].Store; + // Every store the batch does not commit through judges its checks first, each under its own lock, so a refusal or + // contention writes nothing; the home judges its own in CommitBatch. + foreach (var (store, _) in _stores) + { + if (store == home || checks.Length == 0) + continue; + var judged = store.JudgeChecks(checks, wait); + if (judged != CultCommitOutcome.Committed) + return judged; + } + // What the store holds is what the hold resolved: a variant's delta beside its resolved view. - var request = new CultCommitRequest(resolved, deletes, batch.Expected.ToArray(), batch.ExpectsUnchanged); + var request = new CultCommitRequest(resolved, deletes, checks, batch.Expected.ToArray(), batch.ExpectsUnchanged); if (home != null) { if (request.HasConditions && home.IsDirty) @@ -2478,14 +2860,23 @@ namespace GameCult.Caching if (request.HasConditions && _stores.Count > 0) throw new InvalidOperationException("A conditional batch names its home store through a record it upserts or removes."); - var inMemory = _entries.Values - .Select(entry => new CultPersistedRecord { Key = entry.Key.Value, SchemaId = entry.PersistedSchemaId, StoredAt = entry.StoredAt }) - .ToArray(); - return request.ConditionsHold(inMemory, _entries.Values) ? CultCommitOutcome.Committed : CultCommitOutcome.Mismatch; + var inMemory = _entries.Values.ToDictionary(entry => entry.Key.Value, entry => entry.StoredVersion, StringComparer.Ordinal); + return request.ConditionsHold(inMemory, _entries.Values, Array.Empty()) ? CultCommitOutcome.Committed : CultCommitOutcome.Mismatch; }); }); } + // A removal of a key the cache does not hold removes nothing, so no write carries it to a store; but a store that holds a + // record there under an id no registered type declares refuses it, as it refuses every write of that record, rather than let + // the caller believe it was removed. Each store judges it its own way (CacheBackingStore.RefuseRemovalOfUndeclared). + private void RefuseRemovalOfUndeclared(IReadOnlyCollection keys) + { + if (keys.Count == 0) + return; + foreach (var (store, _) in _stores) + store.RefuseRemovalOfUndeclared(keys); + } + // Every add and remove passes here: single writes, committed batches, and loads (source set). // Nothing lands unless all of it is admissible; land is the durable step between validation and memory. private CultCommitOutcome Admit( @@ -2497,12 +2888,47 @@ namespace GameCult.Caching if (_held == null) throw new InvalidOperationException("An admission reached the cache under a plain lock on its gate; every admission runs in a hold."); var home = Validate(admitted, evicted, source); - // The one resolution: a variant admitted, a base admitted or evicted, all resolved here against the post-batch - // set, before land and before memory. land receives the admitted records with variants resolved. - var plan = Resolve(admitted, evicted, source); - var outcome = land(home, plan.Admitted); + // Ids are decided before anything lands, and the caller's documents change only inside this admission: a plan that + // refuses (a duplicate, a malformed id) has changed nothing, and a write that does not commit gives its ids back. + // A write mints random ids; a load of a record written before ids existed mints the ids its reload will mint + // again, and lands in memory only (variants mint inside Resolve, into their override values). + var idPlans = admitted + .Where(stored => stored.Variant == null) + .Select(stored => (Stored: stored, Plan: CultElementIds.Plan(stored.Descriptor.DocumentType, stored.Document, stored.Key.Value, deterministic: source != null))) + .ToArray(); + VariantPlan plan; + CultCommitOutcome outcome; + try + { + foreach (var (_, idPlan) in idPlans) + idPlan.Apply(); + // The one resolution: a variant admitted, a base admitted or evicted, all resolved here against the post-batch + // set, before land and before memory. land receives the admitted records with variants resolved. + plan = Resolve(admitted, evicted, source); + outcome = land(home, plan.Admitted); + } + catch + { + foreach (var (_, idPlan) in idPlans) + idPlan.Undo(); + throw; + } + if (outcome != CultCommitOutcome.Committed) + { + foreach (var (_, idPlan) in idPlans) + idPlan.Undo(); return outcome; + } + + // A record whose ids exist only in memory stays marked until a write persists them (a flush of the whole store + // does, and so does any write of the record: a new entry is unmarked). + if (source != null) + { + foreach (var (stored, idPlan) in idPlans) + stored.IdsInMemoryOnly = idPlan.Count > 0; + } + var applied = Apply(plan, evicted, source); foreach (var change in applied) _held.Changes.Add((change, source != null)); @@ -2518,6 +2944,52 @@ namespace GameCult.Caching // if an observer or an OnUpdate handler throws; then the exception (an AggregateException for several, whatever their // origin: a journal, an observer, a second database) is rethrown. A writer looking for one type unwraps InnerExceptions. // If the body threw, its exception wins. + // Every cache gate entry takes the gate here: a thread that holds a store's gate may not enter a cache's gate it does not + // already hold. User code a cacheless call or an attach runs (a document's getter, a constructor) is the way in; entering a cache + // from it could wait on a thread that holds that cache and waits for this store, so it is refused instead. + private object Gate() + { + if (_storeHeld != null && !Monitor.IsEntered(_gate)) + throw new InvalidOperationException( + "A cache's gate was entered inside a store's hold; user code run by a cacheless call or an attach may not enter another cache."); + return _gate; + } + + // The one lock of a store's gate. A thread holds one store's gate at a time; a store call may be entered again from inside it, + // except a cacheless write: it owns its store outright, so it is the outermost hold of the store's gate. + // A cacheless write that re-enters its own store's hold is refused by whichever fact is true: Cache is the one attachment fact, so + // a store with a cache published (an attach's pull included) gets the attached refusal, and one without gets the re-entry refusal. + internal static T InStoreHold(CacheBackingStore store, Func body, bool cacheless = false) + { + var storeGate = store._storeGate; + if (_storeHeld != null) + { + if (ReferenceEquals(_storeHeld, storeGate)) + { + if (cacheless) + throw new InvalidOperationException(store.Cache != null + ? CacheBackingStore.AttachedRefusal + : "A cacheless write was made inside a call that already holds this store (a store call whose user code made it); write through the cache."); + return body(); + } + throw new InvalidOperationException( + "A store's gate was entered inside another store's hold; user code run by a cacheless call or an attach may not enter a second store."); + } + + lock (storeGate) + { + _storeHeld = storeGate; + try + { + return body(); + } + finally + { + _storeHeld = null; + } + } + } + internal T Held(Func body) { if (Monitor.IsEntered(_gate)) @@ -2534,7 +3006,7 @@ namespace GameCult.Caching var result = default(T)!; try { - lock (_gate) + lock (Gate()) result = body(); } catch (Exception exception) @@ -2681,6 +3153,7 @@ namespace GameCult.Caching if (home.IsReadOnly) throw new InvalidOperationException( $"{stored.Descriptor.SchemaName} record {stored.Key.Value} routes to read-only backing store {home}."); + home.RefuseUndeclared(stored.Key.Value); homes.Add(home); } } @@ -2876,6 +3349,8 @@ namespace GameCult.Caching var type = descriptor.DocumentType; var kept = new List>(); + var overrides = new List(delta.Overrides.Count); + var minted = false; var seen = new HashSet(); foreach (var entry in delta.Overrides) { @@ -2891,9 +3366,10 @@ namespace GameCult.Caching if (member == null) throw new InvalidOperationException($"Variant {key} overrides slot {slot}, which {type.Name} does not have."); + object? value; try { - _codec.Deserialize(member.MemberType, type, entry.Value); + value = _codec.Deserialize(member.MemberType, type, entry.Value); } catch (Exception exception) { @@ -2903,7 +3379,19 @@ namespace GameCult.Caching exception.GetBaseException().Message, exception); } - kept.Add(new KeyValuePair(slot, entry.Value)); + // The override's value is minted like a plain record's members: ids the store never held are filled here, in the + // decoded copy, and the delta this hold lands carries them. + var bytes = entry.Value; + var idPlan = CultElementIds.Plan(member.MemberType, value, key + "." + slot.ToString(CultureInfo.InvariantCulture), deterministic: source != null); + if (idPlan.Count > 0) + { + idPlan.Apply(); + bytes = _codec.Serialize(value!, member.MemberType, type); + minted = true; + } + + overrides.Add(new CultVariantOverride(entry.Op, entry.Path, entry.Id, bytes)); + kept.Add(new KeyValuePair(slot, bytes)); } var payload = _codec.Serialize(baseStored.Document, type, type); @@ -2912,8 +3400,13 @@ namespace GameCult.Caching stack.RemoveAt(stack.Count - 1); return done[key] = isVariant - ? variant.Resolved(document) - : new CultStoredDocument(variant.Key, variant.StoredAt, variant.Descriptor, document) { Flattens = true }; + ? variant.Resolved(document, minted ? new CultVariantDelta(delta.BaseKey, overrides) : delta, variant.IdsInMemoryOnly || (minted && source != null), _codec) + : new CultStoredDocument(variant.Key, variant.StoredAt, variant.Descriptor, document) + { + Flattens = true, + StoredSchemaId = variant.StoredSchemaId, + StoredVersion = variant.StoredVersion + }; } } @@ -3089,7 +3582,7 @@ namespace GameCult.Caching private T? Single(string lookup, Func> keys) where T : class { - lock (_gate) + lock (Gate()) { var found = keys().Distinct().ToArray(); if (found.Length > 1) @@ -3204,6 +3697,9 @@ namespace GameCult.Caching protected CultDocumentRegistry Registry => _registry ?? throw new InvalidOperationException("Backing store is not attached to a CultDocumentRegistry."); + // Null for a store no registry was attached to (CultMesh's raw single-file write). + private protected CultDocumentRegistry? AttachedRegistry => _registry; + protected ConcurrentDictionary Entries { get; } = new(StringComparer.Ordinal); @@ -3215,48 +3711,163 @@ namespace GameCult.Caching public IReadOnlyList LastSchemaMigrationReports => _lastSchemaMigrationReports; + /// + /// The records this store's last load found and carries untouched, ordered by key: no registered type owns their schema id or + /// lists it as compatible. See . + /// + public IReadOnlyList ForeignRecords => _foreignRecords; + + private CultForeignRecord[] _foreignRecords = Array.Empty(); + // Set by the cache at attach. A pull hands over everything it loaded and dropped in one call before it adopts // any of it; if the cache refuses a record the call throws and the store keeps its previous view. protected internal Action, IReadOnlyList>? Loaded; - // Set by the cache at attach: the cache's own resolution (variants, their indexed values, their bases) run over a - // set of records a store is about to hold, mutating nothing. It throws the refusal the cache would give the same - // set arriving as a load. - protected internal Action, IReadOnlyList>? Judging; + internal readonly object _storeGate = new(); - private readonly object _detachedGate = new(); + // Gates the current thread has taken for store calls, read by tests (InternalsVisibleTo) to pin the two bounds of Gate: at + // most two under failing-attach churn (Cache read under the store's gate is then always null), at most three otherwise: a + // stale non-null guess (its gate), the store's gate, and the settled cache's gate. + [ThreadStatic] + internal static int GateTakes; + + internal const string AttachedRefusal = "The store is attached to a cache, write through the cache."; + // Written only by CultCache.AddBackingStore, holding the cache's gate and this store's: set before the pull, and set back to null + // in the same hold if the pull fails. Read under this store's gate it is settled, and a settled cache never changes, because + // nothing detaches. internal CultCache? Cache; - // Once attached, a hold on the cache's gate: the store and its cache share one lock, so a load calling back into - // the cache can never take the gate after the store lock, and a direct call publishes what it loaded when it - // returns. File locks are always taken inside it. - protected T Held(Func body) + private const string DirectoryFormatV4 = "cultcache.store.v4.directory-content-addressed-pages"; + private const string DirectoryFormatV5 = "cultcache.store.v5.directory-content-addressed-pages"; + + /// + /// The one header decision, for every writer of every store: the string a store file's format header must carry after + /// this write. + /// + /// + /// The marked header (v3, or the directory store's v5) says the store holds an element id: a reader older than ids refuses + /// it, because it would skip the id slots and rewrite the elements without them. A writer that rewrites the whole store + /// () decides by content, so a store drops back to unmarked when nothing in it holds an id. + /// A writer that touches only part of the store keeps a header already marked (), because + /// it cannot see what the rest holds. is about the records this write knows are on disk. + /// Unmarked, a single-file store is v2 exactly when it holds a variant () and otherwise v1; + /// the directory store refuses variants and is v4. Pass the header the file carries, or null when it has none, and refuse + /// a file whose header you cannot read before calling this. + /// + protected internal static string HeaderFor(bool holdsIds, string? existingHeader, bool wholeStore, bool directoryStore, bool holdsVariants = false) { - if (Cache is { } cache) - return cache.Held(body); - lock (_detachedGate) - return body(); + var marked = directoryStore ? DirectoryFormatV5 : CultPersistedStoreSnapshot.FormatV3; + if (holdsIds || (!wholeStore && string.Equals(existingHeader, marked, StringComparison.Ordinal))) + return marked; + return directoryStore ? DirectoryFormatV4 : holdsVariants ? CultPersistedStoreSnapshot.FormatV2 : CultPersistedStoreSnapshot.FormatV1; } + // The header a write leaves, for either store kind: HeaderFor decides. An element id is an ordinary keyed string, so only a + // reader with the record's type can tell from its bytes whether it holds one; the store header is how a reader without the types + // learns it. A write that copies any + // record forward keeps a marked header; a write that encodes every record the store will hold decides by content. An + // opaque payload (null) never marks an unmarked store, and keeps a marked one marked. A copied record the writer decodes + // under the store's lock holding an id marks the store too (SingleFileBackingStore.CopiedHoldIds): another runtime may + // have written ids under an unmarked header. Nothing a load or a decode records can unmark a store. + protected static string HeaderForWrite(string? durableHeader, IReadOnlyCollection written, bool copiesAny, bool copiedHoldIds, bool directoryStore, bool holdsVariants) => + HeaderFor( + holdsIds: copiedHoldIds || written.Any(value => value == true), + durableHeader, + wholeStore: !copiesAny && written.All(value => value != null), + directoryStore, + holdsVariants); + + // Every store call comes through here, and this is the one owner of which gate it holds. Cache is read once, unlocked, as a + // guess that only picks which gate to try. The store's gate then settles Cache: an attach publishes it and takes it back inside + // one hold of that gate, so under it Cache is what it will always be, and a settled cache is permanent because nothing detaches. + // The settled cache's gate therefore cannot miss. A call takes at most three gates: the guess's, the store's, the settled cache's. + // Attached, the call holds the cache's gate: the store and its cache share one lock, so a load calling back into the cache never + // takes the gate after the store's, and a direct call publishes what it loaded when it returns. Detached, it holds the store's + // gate; a cacheless call (WriteRecord, WriteDocument) that finds a cache there is refused in that same hold, and one that + // re-enters a hold of this store (a cacheless write's user code) is refused by InStoreHold before it reads Cache. A call during + // an attach, from the attaching thread, is the journal entering its cache and is refused. File locks are always taken inside + // either gate. + private T Gate(Func body, bool cacheless) + { + var guess = cacheless ? null : Volatile.Read(ref Cache); + if (guess != null) + { + GateTakes++; + var (ran, result) = guess.Held(() => Cache == guess ? (true, body()) : (false, default(T)!)); + if (ran) + return result; + } + + GateTakes++; + var (done, value, settled) = UnderStoreGate(() => + { + var cache = Cache; + if (cache == null) + return (true, body(), (CultCache?)null); + if (cacheless) + throw new InvalidOperationException(AttachedRefusal); + return (false, default(T)!, cache); + }, cacheless); + if (done) + return value; + + GateTakes++; + return settled!.Held(() => Cache == settled + ? body() + : throw new InvalidOperationException( + "A store's settled cache changed; the store's cache is written only by an attach, inside the store's gate.")); + } + + protected T Held(Func body) => Gate(body, cacheless: false); + protected void Held(Action body) => Held(() => { body(); return true; }); - internal void AttachRegistry(CultDocumentRegistry registry) - { - _registry = registry ?? throw new ArgumentNullException(nameof(registry)); - } + // A cacheless call. The registry is the call's argument, not store state: it is attached for the call and detached when the + // call ends, whatever it does. A cacheless store holds none between calls. + private protected void Cacheless(CultDocumentRegistry? registry, Action body) => + Gate(() => + { + UseRegistry(registry); + try + { + body(); + } + finally + { + UseRegistry(null); + } + return true; + }, cacheless: true); + + // The store's own gate, taken only here. Lock order: a cache's gate, then a store's; nothing takes a store's gate and then a cache's. + internal T UnderStoreGate(Func body, bool cacheless = false) => CultCache.InStoreHold(this, body, cacheless); + + // The one writer of the lent registry: a cacheless call for the call, an attach for as long as it holds the store, and a failed + // attach puts it back, so a cacheless store holds none between calls and a pull on one loads nothing. Nothing else is lent, and the + // pull reads the lent registry, not Cache. + internal void UseRegistry(CultDocumentRegistry? registry) => _registry = registry; public abstract void PullAll(); public abstract void Push(CultStoredDocument entry); public abstract void Delete(CultStoredDocument entry); public abstract void PushAll(); - // One durable step under the store's exclusive lock; conditions are evaluated against what is on disk then. - // A failure restores the store's staged view. + /// + /// One durable step under the store's exclusive lock: the batch and every staged write land onto the store as it is then, + /// and the batch's conditions are evaluated against it. A failure restores the store's staged view. + /// + /// + /// An implementation judges , and its own , under that + /// lock with the keys it writes: a key whose record is under an id no registered type declares refuses the commit, and a + /// checked key is never removed. The cache calls on every other store before this commit. Each + /// foreign record a store lists comes from with the version it read, so pass to + /// . + /// public abstract CultCommitOutcome CommitBatch(CultCommitRequest request, bool wait); public virtual void Dispose() { - if (FlushOnDispose && IsDirty && !IsReadOnly) + if (FlushOnDispose && (IsDirty || HasStagedChecks) && !IsReadOnly) { PushAll(); } @@ -3285,15 +3896,18 @@ namespace GameCult.Caching // resolution shed. A plain record sheds a member its type dropped by never carrying it into the document; a variant // sheds the override of such a slot by never carrying it into the delta, so a later type reusing the slot number // cannot resurrect it and a flush needs no rule of its own. The report is the schema diff's ignored slots plus the - // overrides whose slot the type does not have, including one the persisted schema never listed. + // overrides whose slot the type does not have, including one the persisted schema never listed. The version is the reading + // store's: it read the bytes. protected CultStoredDocument ToStoredDocument( CultPersistedRecord record, + string version, IReadOnlyCollection catalog, Func deserializePayload, out CultSchemaMigrationReport report) { var resolution = Registry.ResolvePersistedSchemaDetailed(record.SchemaId, catalog); report = resolution.Report; + var readWhole = report.IgnoredExtraSlots.Length == 0; if (record.Variant != null) { var descriptor = resolution.Descriptor; @@ -3317,14 +3931,21 @@ namespace GameCult.Caching })).ToArray(); } - // The cache resolves it against its base when it admits the load. + // The store holds the delta and the cache resolves it against its base, with its own codec, when it admits the load. A + // codecless variant counts as holding ids, but a header reads HoldsIds only on staged entries, which are always the cache's + // pushed documents. return new CultStoredDocument( new CultRecordKey(record.Key), record.StoredAt, descriptor, new CultVariantDelta(record.Variant.BaseKey, record.Variant.Overrides.Where(Has).ToArray()), null, - record.SchemaId); + null) + { + StoredSchemaId = record.SchemaId, + StoredSchemaReadWhole = readWhole, + StoredVersion = version + }; } object document; @@ -3343,8 +3964,12 @@ namespace GameCult.Caching new CultRecordKey(record.Key), record.StoredAt, resolution.Descriptor, - document, - record.SchemaId); + document) + { + StoredSchemaId = record.SchemaId, + StoredSchemaReadWhole = readWhole, + StoredVersion = version + }; } protected void SetLastSchemaMigrationReports(IEnumerable reports) @@ -3352,27 +3977,232 @@ namespace GameCult.Caching _lastSchemaMigrationReports = reports?.ToArray() ?? Array.Empty(); } - protected void MarkFlushSucceeded() + // The one test of whether a stored record is foreign: no registered type claims its schema id through the catalog it was + // stored with. It reads the record's header only, never its payload. Null for a record the registry resolves. + // The one constructor path for a foreign record, so every store kind, one outside this library included, lists it at the + // version it read. + protected CultForeignRecord? Foreign(CultPersistedRecord record, IReadOnlyCollection catalog, string storedVersion) => + Registry.Claimant(record.SchemaId, catalog, out var persisted) == null + ? new CultForeignRecord(record.Key, record.SchemaId, persisted!.SchemaName, record.StoredAt, storedVersion) + : null; + + protected void SetForeignRecords(IEnumerable foreign) { - IsDirty = false; + _foreignRecords = foreign.OrderBy(record => record.Key, StringComparer.Ordinal).ToArray(); + } + + // A cache never writes, relabels or removes a record it does not own: the staging half of rule 3. Refused when the store + // carries a foreign record at the key, or holds one that is not owned: its stored id is not declared by a registered type and + // the type that reads it drops persisted slots (read-only). A record read whole is owned; ToStoredDocument decided that from + // the same comparison Owns makes. Called for every record a cache write admits or evicts, before anything is staged. + protected internal void RefuseUndeclared(string key) + { + var foreign = _foreignRecords.FirstOrDefault(record => string.Equals(record.Key, key, StringComparison.Ordinal)); + if (foreign != null) + throw Overwrites(null, new[] { (foreign.Key, foreign.SchemaId, foreign.SchemaName) }); + if (Entries.TryGetValue(key, out var held) && !Registry.Declared(held.StoredSchemaId) && !held.StoredSchemaReadWhole) + throw Overwrites(null, new[] { (key, held.StoredSchemaId, held.Descriptor.SchemaName) }); } /// - /// Records that this store's completed durable write stored under its descriptor, so its - /// is again. Call it - /// under Held, once the write is durable, for each record the write rewrote; a record whose bytes the write - /// kept is not marked. A rewrite left unmarked makes conditional commit compare the schema id the record was read - /// under, which fails closed until the next pull. + /// Called when the cache removes keys it does not hold, outside a batch that commits through this store. Such a removal + /// removes nothing, but a store that holds a record at one of these keys under an id no registered type declares must + /// refuse it rather than let the caller believe the record is gone. /// - /// is not the entry this store holds in - /// at its key: a store marks only its own rewrite. - internal void MarkWritten(CultStoredDocument entry) + /// + /// + /// The default refuses now, from this store's view: the foreign records its last load listed and the records it holds + /// (). It reads nothing, so it cannot see a record another writer put there after that load. + /// + /// + /// A writable store kind that reads its durable state under a lock overrides this to call , and + /// judges at its next or as it judges + /// : under the lock, with the keys it writes, refused through the same refusal, never + /// removed. It calls when that step succeeds, which clears them. A read-only store has no + /// such step and keeps the default. Both shipped store kinds do this. A batch's own checks are judged through + /// . + /// + /// + protected internal virtual void RefuseRemovalOfUndeclared(IReadOnlyCollection keys) { - if (entry == null) - throw new ArgumentNullException(nameof(entry)); - if (!Entries.TryGetValue(entry.Key.Value, out var held) || !ReferenceEquals(held, entry)) - throw new ArgumentException($"Record '{entry.Key.Value}' is not the entry this store holds; a store marks only what it wrote.", nameof(entry)); - entry.MarkWritten(); + foreach (var key in keys) + RefuseUndeclared(key); + } + + /// + /// Called for a batch's checks on every attached store that is not the batch's home, before the home commits. It judges + /// only the keys it is given: it never writes, stages, clears or calls + /// . + /// + /// + /// The default answers from this store's view () and reads nothing, which is right for a + /// read-only store. A writable store kind that reads its durable state under a lock overrides this to read under that + /// lock and refuse through , returning when it may not wait + /// and the lock is held. A refusal or contention here writes nothing anywhere, because the home has not yet committed. + /// + protected internal virtual CultCommitOutcome JudgeChecks(IReadOnlyCollection keys, bool wait) + { + foreach (var key in keys) + RefuseUndeclared(key); + return CultCommitOutcome.Committed; + } + + // Keys a removal named that this store's cache does not hold, waiting for this store's next flush or commit to judge them. + // A refusal takes back the keys it refused; a successful flush or commit clears them. + private readonly HashSet _checks = new(StringComparer.Ordinal); + + /// + /// The keys staged for this store's next flush or commit to judge. They are not writes: + /// does not count them. + /// + protected IReadOnlyCollection StagedChecks => _checks; + + /// + /// Stages keys for this store's next flush or commit to judge without removing them. See + /// . + /// + protected void StageChecks(IEnumerable keys) => Held(() => + { + foreach (var key in keys) + _checks.Add(key); + }); + + internal bool HasStagedChecks => _checks.Count > 0; + + // A stored record under an id the writer does not own, named by the catalog entry that publishes the id. Header only: no + // payload is decoded. Null for a record under an owned id. + protected static (string Key, string SchemaId, string SchemaName)? Undeclared( + CultPersistedRecord record, IReadOnlyCollection catalog, Func, bool> owns) => + owns(record.SchemaId, catalog) + ? null + : (record.Key, record.SchemaId, CultDocumentRegistry.PublishedBy(record.SchemaId, catalog)?.SchemaName ?? string.Empty); + + // The apply half of rule 3, for every store kind, checked where the write lands under the store's lock. A key this write names + // whose durable record is under an id no registered type declares is refused, every such key at once, through Refuse, which + // reloads it as the store holds it: listed foreign when no type reads it, held read-only when one does. + protected void RefuseUndeclaredOnDisk( + IEnumerable written, + Func undeclaredAt, + Func durableAt, + Func foreignAt, + Action unstage, + bool cacheless = false) + { + var refused = written.Distinct(StringComparer.Ordinal) + .Select(undeclaredAt) + .OfType<(string Key, string SchemaId, string SchemaName)>() + .OrderBy(record => record.Key, StringComparer.Ordinal) + .ToArray(); + if (refused.Length == 0) + return; + Refuse(refused.Select(record => record.Key), durableAt, foreignAt, unstage, inner => Overwrites(inner, refused, cacheless)); + } + + /// + /// The one refusal of a write, for every store kind: it reloads what it refuses, or changes nothing. For each key, + /// gives the durable record decoded as a load would (null when the store holds none, or holds + /// it as a foreign record), and the foreign record it holds there, if any. What differs from this + /// store's view is handed to , the path a pull uses; only when that succeeds does the store forget its + /// staged change at each key (), take the loaded and dropped records into its view, and list the + /// keys' foreign records as a pull lists them. If the reload throws, nothing has changed and the refusal carries that + /// exception. Always throws what returns. + /// + protected void Refuse( + IEnumerable keys, + Func durableAt, + Func foreignAt, + Action unstage, + Func refusal) + { + var refused = keys.Distinct(StringComparer.Ordinal).OrderBy(key => key, StringComparer.Ordinal).ToArray(); + var loaded = new List(); + var dropped = new List(); + var foreign = new List(); + try + { + foreach (var key in refused) + { + if (foreignAt(key) is { } carried) + foreign.Add(carried); + Entries.TryGetValue(key, out var held); + var durable = durableAt(key); + if (durable == null) + { + if (held != null) + dropped.Add(held); + } + else if (held == null || !SameStoredIdentity(held.StoredVersion, durable.StoredVersion)) + loaded.Add(durable); + } + + if (loaded.Count > 0 || dropped.Count > 0) + Loaded?.Invoke(loaded, dropped); + } + catch (Exception ex) + { + throw refusal(ex); + } + + foreach (var key in refused) + { + unstage(key); + _checks.Remove(key); + } + foreach (var held in dropped) + Entries.TryRemove(held.Key.Value, out _); + foreach (var stored in loaded) + Entries[stored.Key.Value] = stored; + SetForeignRecords(_foreignRecords.Where(record => !refused.Contains(record.Key, StringComparer.Ordinal)).Concat(foreign)); + throw refusal(null); + } + + // Two stored records are the same stored bytes when their versions are equal. The one place a pull's reload filter, the + // staleness check and a refusal's reload decide it. + protected static bool SameStoredIdentity(string? version, string? otherVersion) => + string.Equals(version, otherVersion, StringComparison.Ordinal); + + // The one spelling of a version in both store kinds: the lowercase hex of the SHA-256 of a record's persisted encoding. + protected static string VersionOf(byte[] sha256) + { + var hex = new StringBuilder(sha256.Length * 2); + foreach (var value in sha256) + hex.Append(value.ToString("x2", CultureInfo.InvariantCulture)); + return hex.ToString(); + } + + // The one re-mint, for every store kind (storedat-on-change): a record written over one the store holds is stored at a storedAt + // later than the one it replaces, whoever wrote that. The staged storedAt stands only when it is already later (both parsed + // RoundtripKind; an unparsable one counts as not later); otherwise it is minted from the replaced one. + protected string StoredAtOver(string storedAt, string replacedAt) => + StoredLater(storedAt, replacedAt) ? storedAt : CultCache.MintStoredAt(replacedAt, (Cache?.Clock ?? (() => DateTimeOffset.UtcNow))()); + + private static bool StoredLater(string storedAt, string than) => + DateTimeOffset.TryParse(storedAt, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind, out var later) && + DateTimeOffset.TryParse(than, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind, out var earlier) && + later > earlier; + + // The one refusal of a write onto records the writer does not own, carried foreign or held read-only. + // It names each record's key, stored id and schema name, never its payload. When the reload that refusal owes failed, the + // refusal carries that failure as its inner exception. + // A cacheless writer (a CultMesh single-file write) owns only what the catalog entry it writes declares or reads whole, and has nothing to reload. + protected CultSchemaConflictException Overwrites(Exception? inner, (string Key, string SchemaId, string SchemaName)[] refused, bool cacheless = false) => new( + $"{(refused.Length == 1 ? "Record" : "Records")} {string.Join(", ", refused.Select(record => $"'{record.Key}' (schema id '{record.SchemaId}', '{record.SchemaName}')"))} in {this}: " + + (cacheless + ? "the catalog entry this write publishes neither declares this schema id nor reads it whole, so the write never relabels or removes it; nothing was written and nothing was reloaded. " + + "List the id as compatible on the schema that should own it to accept the loss of its unread slots." + : "no registered type owns or declares this schema id, or the type that reads it would drop persisted slots, so this cache never writes, relabels or removes it. " + + "Declare the id compatible on the type that should own it to accept the loss.") + + (inner == null ? string.Empty : " Reloading what the write refused failed, so the cache is unchanged."), + refused[0].SchemaId, + refused.Select(record => record.SchemaName).Distinct(StringComparer.Ordinal).ToArray(), + refused[0].Key, + inner); + + // A flush or commit succeeded: nothing staged is left, writes or checks. + protected void MarkFlushSucceeded() + { + IsDirty = false; + _checks.Clear(); } } @@ -3385,10 +4215,89 @@ namespace GameCult.Caching } protected FileInfo FileInfo { get; } - protected abstract byte[] SerializeSnapshot(CultPersistedStoreSnapshot snapshot); - protected abstract CultPersistedStoreSnapshot DeserializeSnapshot(byte[] data); - protected abstract byte[] SerializePayload(object document); + // Keys this cache has written or removed here since the store last wrote them: the only keys a foreign record can refuse. + // A key is staged exactly while the store is dirty with it, and a pull runs only on a clean store. + private readonly HashSet _staged = new(StringComparer.Ordinal); + + // The version of every record the store held when this store last read it, as this store last wrote it for keys it wrote. + // A write into a store holding a variant lands only while the durable versions still equal these. + private readonly Dictionary _lastRead = new(StringComparer.Ordinal); + + /// + /// A store file from its parts, each already encoded: a catalog entry or record this store carries forward is the bytes its + /// file held, one it writes is or of the object. + /// + protected abstract byte[] SerializeStore(string formatVersion, IReadOnlyList catalogEntries, IReadOnlyList records); + + // One record's persisted encoding: the bytes a version names, and the bytes a written record is stored as. + protected abstract byte[] SerializeRecord(CultPersistedRecord record); + + // One catalog entry's persisted encoding, for an entry the writer built. + protected abstract byte[] SerializeCatalogEntry(CultSchemaCatalogEntry entry); + + /// + /// This store's one reader of a non-empty store file, and so its one verdict on whether the file may be replaced: open, + /// flush and commit all ask it. It must throw for anything that is not exactly one + /// store this runtime reads (a truncated file, bytes after the store, a missing or extra slot, a format or record it does + /// not know), so a rewrite never overwrites a file it cannot see. It returns the decoded snapshot and, parallel to the + /// snapshot's catalog and records, the exact bytes the file holds for each: the store keeps those, never the snapshot's objects. + /// + protected abstract (CultPersistedStoreSnapshot Snapshot, byte[][] CatalogBytes, byte[][] RecordBytes) ReadStore(byte[] data); + + // Test seam: how many record encodings this store has hashed, to prove what a write hashes. + internal int RecordsHashed; + + // Test seam: how many copied records a write decoded to ask whether they hold an element id. + internal int CopiedRecordsDecoded; + + private string VersionOfBytes(byte[] bytes) + { + RecordsHashed++; + using var sha256 = SHA256.Create(); + return VersionOf(sha256.ComputeHash(bytes)); + } + + // What the store's file holds, as the store owns it: the snapshot as read, and keyed by record key the exact slice the file + // holds for each record, whose version is the SHA-256 of that slice, hashed when a comparison first asks. Catalog entries + // keep their slices by the entry object this read produced. A write copies a record it did not stage from its slice here + // and re-encodes every record it was handed, so no object a caller edits is ever mistaken for the bytes in the file. + private sealed class Durable + { + private readonly SingleFileBackingStore _owner; + private readonly Dictionary _records; + private readonly Dictionary _versions = new(StringComparer.Ordinal); + private readonly Dictionary _catalog = new(); + + internal Durable(SingleFileBackingStore owner, (CultPersistedStoreSnapshot Snapshot, byte[][] CatalogBytes, byte[][] RecordBytes) read) + { + _owner = owner; + Snapshot = read.Snapshot; + _records = new Dictionary(StringComparer.Ordinal); + for (var index = 0; index < Snapshot.Records.Length; index++) + _records[Snapshot.Records[index].Key] = read.RecordBytes[index]; + for (var index = 0; index < Snapshot.SchemaCatalog.Length; index++) + _catalog.Add(Snapshot.SchemaCatalog[index], read.CatalogBytes[index]); + } + + internal CultPersistedStoreSnapshot Snapshot { get; } + + internal byte[] Slice(string key) => _records[key]; + + internal string Version(string key) + { + if (!_versions.TryGetValue(key, out var version)) + _versions[key] = version = _owner.VersionOfBytes(_records[key]); + return version; + } + + // The bytes to write for a catalog entry the write chose: the slice the file held when it is one of this read's entries, + // otherwise an encoding of the entry the writer built. + internal byte[] CatalogBytes(CultSchemaCatalogEntry entry) => + _catalog.TryGetValue(entry, out var bytes) ? bytes : _owner.SerializeCatalogEntry(entry); + } + + protected abstract byte[] SerializePayload(object document, Type documentType); protected abstract object DeserializePayload(Type documentType, byte[] payload); public override string ToString() => FileInfo.FullName; @@ -3402,26 +4311,37 @@ namespace GameCult.Caching if (IsDirty) return; - var snapshot = ReadSnapshot(); - if (snapshot == null) + var durable = ReadSnapshot(); + if (durable == null) { + _lastRead.Clear(); SetLastSchemaMigrationReports(Array.Empty()); + SetForeignRecords(Array.Empty()); return; } + var snapshot = durable.Snapshot; var reports = new List(snapshot.Records.Length); var persisted = new Dictionary(StringComparer.Ordinal); + var foreign = new List(); + var versions = new Dictionary(StringComparer.Ordinal); foreach (var record in snapshot.Records) { - var stored = ToStoredDocument(record, snapshot.SchemaCatalog, DeserializePayload, out var report); + var version = versions[record.Key] = durable.Version(record.Key); + if (Foreign(record, snapshot.SchemaCatalog, version) is { } carried) + { + foreign.Add(carried); + continue; + } + + var stored = ToStoredDocument(record, version, snapshot.SchemaCatalog, DeserializePayload, out var report); reports.Add(report); persisted[stored.Key.Value] = stored; } var loaded = persisted.Values .Where(stored => !Entries.TryGetValue(stored.Key.Value, out var existing) || - existing.StoredAt != stored.StoredAt || - existing.PersistedSchemaId != stored.PersistedSchemaId) + !SameStoredIdentity(existing.StoredVersion, stored.StoredVersion)) .ToArray(); var dropped = Entries.Values.Where(existing => !persisted.ContainsKey(existing.Key.Value)).ToArray(); if (loaded.Length > 0 || dropped.Length > 0) @@ -3431,7 +4351,11 @@ namespace GameCult.Caching Entries.TryRemove(stored.Key.Value, out _); foreach (var stored in loaded) Entries[stored.Key.Value] = stored; + _lastRead.Clear(); + foreach (var (key, version) in versions) + _lastRead[key] = version; SetLastSchemaMigrationReports(reports); + SetForeignRecords(foreign); } public override void Push(CultStoredDocument entry) @@ -3440,6 +4364,7 @@ namespace GameCult.Caching Held(() => { Entries[entry.Key.Value] = entry; + _staged.Add(entry.Key.Value); IsDirty = true; }); } @@ -3450,12 +4375,16 @@ namespace GameCult.Caching Held(() => { Entries.TryRemove(entry.Key.Value, out _); + _staged.Add(entry.Key.Value); IsDirty = true; }); } - // A plain flush writes this store's whole view under the lock: two writers never interleave bytes, but the last - // one wins. Processes sharing a store must use conditional commit. + // A flush applies this store's staged writes and removals onto what the file holds now, under the lock: two writers never + // interleave bytes, every record this cache did not stage stays as the file has it, and a plain flush is last-writer-wins + // only per staged key. A key whose durable record is under an id this cache does not declare refuses the write, and in a store + // holding a variant it lands only onto the store as this cache last read it (both ApplyWriteSet). + // A file it cannot read (truncated, trailing bytes, not a store, a format it does not read) is refused and left as it is. public override void PushAll() { ThrowIfReadOnly(); @@ -3463,14 +4392,12 @@ namespace GameCult.Caching { using (AcquireLock(wait: true)) { - RefuseToRewriteAStoreItCannotRead(); - WriteSnapshot( - Entries.Values.Select(entry => ToPersistedRecord(entry, SerializePayload)), - Entries.Values.Select(entry => entry.Descriptor.ToCatalogEntry())); + var disk = ReadSnapshot(); + var (upserts, removals) = StagedWrites(Array.Empty(), Array.Empty()); + Wrote(ApplyWriteSet(disk, upserts, removals, Array.Empty())); } - foreach (var entry in Entries.Values) - MarkWritten(entry); + _staged.Clear(); MarkFlushSucceeded(); }); } @@ -3481,145 +4408,494 @@ namespace GameCult.Caching return Held(() => CommitBatchCore(request, wait)); } + // Conditions are checked against the file under the lock, and only the condition differs between a conditional and an + // unconditional commit: both apply the batch and every staged write onto the file as it is. private CultCommitOutcome CommitBatchCore(CultCommitRequest request, bool wait) { using var fileLock = AcquireLock(wait); if (fileLock == null) return CultCommitOutcome.Contended; - var disk = ReadSnapshot() ?? new CultPersistedStoreSnapshot(); - if (!request.ConditionsHold(disk.Records, Entries.Values)) + var disk = ReadSnapshot(); + if (request.HasConditions && !request.ConditionsHold( + (disk?.Snapshot.Records ?? Array.Empty()).ToDictionary(record => record.Key, record => disk!.Version(record.Key), StringComparer.Ordinal), + Entries.Values, + ForeignRecords)) return CultCommitOutcome.Mismatch; - // An unconditional commit writes what a flush would, this store's whole view plus the batch: last-writer-wins, - // and staged single writes land with it. A conditional commit (store clean) lands the batch onto the file as it is. - var ontoDisk = request.HasConditions && !IsDirty; - var records = ontoDisk - ? disk.Records.ToDictionary(record => record.Key, StringComparer.Ordinal) - : Entries.Values.Select(entry => ToPersistedRecord(entry, SerializePayload)).ToDictionary(record => record.Key, StringComparer.Ordinal); - var catalog = ontoDisk - ? disk.SchemaCatalog - : Entries.Values.Select(entry => entry.Descriptor.ToCatalogEntry()); - // The committer judged the batch against its own view; the file may have moved since. A merge is judged on the - // set the file will hold, exactly as that set would be judged loading: what other processes wrote (arriving), - // what they removed (departing), and this batch. - if (ontoDisk && Judging != null && MergeInvolvesVariants(request, disk)) - JudgeMerge(request, disk); - foreach (var entry in request.Deletes) - records.Remove(entry.Key.Value); - foreach (var entry in request.Upserts) - records[entry.Key.Value] = ToPersistedRecord(entry, SerializePayload); - - WriteSnapshot(records.Values, catalog.Concat(request.Upserts.Select(entry => entry.Descriptor.ToCatalogEntry()))); + var (upserts, removals) = StagedWrites(request.Upserts, request.Deletes); + var written = ApplyWriteSet(disk, upserts, removals, request.Checks); foreach (var entry in request.Deletes) Entries.TryRemove(entry.Key.Value, out _); foreach (var entry in request.Upserts) Entries[entry.Key.Value] = entry; - // The whole view was written, each record under its own descriptor; onto the disk, the rest kept their bytes. - if (!ontoDisk) - { - foreach (var entry in Entries.Values) - MarkWritten(entry); - } - + Wrote(written); + _staged.Clear(); MarkFlushSucceeded(); return CultCommitOutcome.Committed; } - // Only a variant makes a merge more than the committer's own judgement: a variant it lands, one another writer wrote - // (read from the record's header, never its payload), or one this cache holds. A plain-only store never decodes - // another writer's records, so a schema this cache does not register cannot fail a commit that involves no variant. - private bool MergeInvolvesVariants(CultCommitRequest request, CultPersistedStoreSnapshot disk) => - Cache?.HoldsVariants == true || - request.Upserts.Any(entry => entry.Variant != null) || - disk.Records.Any(record => record.Variant != null); - - private void JudgeMerge(CultCommitRequest request, CultPersistedStoreSnapshot disk) + // What a write puts in the store and takes out of it: the staged keys as this cache holds them now (a key it no longer holds + // is a removal), and a commit's batch, which wins over a staged write of the same key. Records it did not stage are not here. + private (List Upserts, List Removals) StagedWrites( + IReadOnlyList batchUpserts, + IReadOnlyList batchDeletes) { - var landing = request.Upserts.Select(entry => entry.Key.Value).ToHashSet(StringComparer.Ordinal); - var removing = request.Deletes.Select(entry => entry.Key.Value).ToHashSet(StringComparer.Ordinal); - var onDisk = disk.Records.Select(record => record.Key).ToHashSet(StringComparer.Ordinal); - var arriving = new List(request.Upserts); - foreach (var record in disk.Records) + var documents = new Dictionary(StringComparer.Ordinal); + var removals = new HashSet(StringComparer.Ordinal); + foreach (var key in _staged) { - if (landing.Contains(record.Key) || removing.Contains(record.Key)) - continue; - if (Entries.TryGetValue(record.Key, out var known) && known.StoredAt == record.StoredAt && known.PersistedSchemaId == record.SchemaId) - continue; - arriving.Add(ToStoredDocument(record, disk.SchemaCatalog, DeserializePayload, out _)); + if (Entries.TryGetValue(key, out var held)) + documents[key] = held; + else + removals.Add(key); } - var departing = new List(request.Deletes); - departing.AddRange(Entries.Values.Where(known => - !onDisk.Contains(known.Key.Value) && !landing.Contains(known.Key.Value) && !removing.Contains(known.Key.Value))); - Judging!(arriving, departing); + foreach (var entry in batchDeletes) + { + documents.Remove(entry.Key.Value); + removals.Add(entry.Key.Value); + } + + foreach (var entry in batchUpserts) + { + documents[entry.Key.Value] = entry; + removals.Remove(entry.Key.Value); + } + + var catalogEntries = new Dictionary(); + var upserts = documents.Values.Select(document => + { + if (!catalogEntries.TryGetValue(document.Descriptor, out var catalogEntry)) + catalogEntries[document.Descriptor] = catalogEntry = document.Descriptor.ToCatalogEntry(); + return new CultStagedRecord(ToPersistedRecord(document, payload => SerializePayload(payload, payload.GetType())), catalogEntry, document.HoldsIds); + }).ToList(); + return (upserts, removals.ToList()); } - // A plain flush replaces the file without merging it, so under the lock it first reads the disk the way attach and - // commit do: a store whose stamp this runtime does not read is refused and left as it is, and so is a file that - // cannot be read (an I/O or access error). A file whose stamp reads as v1 or v2, or cannot be read at all (empty, - // truncated), is not a newer store, and the flush proceeds as it always has, so it can still heal; that includes a - // v1 file whose body is corrupt or truncated. - private void RefuseToRewriteAStoreItCannotRead() + /// + /// Writes one record to the store file without a cache. Under the store's lock the file is read by the same reader a cache's + /// store uses, replaces the record at its key, and every other record and catalog entry is copied + /// forward as stored. The file is replaced atomically, and its format header is decided as a flush decides it: + /// true marks the store as holding element ids, false leaves it unmarked, and null (a payload + /// the caller cannot read) keeps whatever the file already says. A write that finds the file unreadable, or a record at + /// the key under a schema id does not declare, is refused and leaves the file as it was. + /// + /// The record to store. + /// The schema catalog entry for the record's schema id. + /// Whether the record's payload holds element ids, or null when the caller cannot tell. + /// The registry the payload was encoded with. Given, the write decides the header for copied records that registry can read, and its other types also claim stored records as a cache registering them would; it declares nothing, so it can only refuse more. stays the only description of its own schema id and the registry's type for that id is never asked. Owned are a record whose id declares (its own or one it lists compatible) and a record the entry reads whole when it is the picked claimant; refused are a record another registered type claims, one whose stored entry lists another registered type's id ahead of the entry's, and one that two registered types list or two versions of the schema could read. Omitted, it decides nothing for copied records and the one entry is the only claimant. It serves this call only: the store keeps no registry. + /// The call holds the store's gate while it runs. User code it runs must not enter a cache or another store; that throws . + /// The store is attached to a cache; write through the cache. + /// The file at the path is not a store this runtime can read. + public void WriteRecord(CultPersistedRecord record, CultSchemaCatalogEntry catalogEntry, bool? holdsIds = null, CultDocumentRegistry? registry = null) { - var bytes = ReadFileBytes(); - if (bytes == null) + if (record == null) throw new ArgumentNullException(nameof(record)); + if (catalogEntry == null) throw new ArgumentNullException(nameof(catalogEntry)); + Cacheless(registry, () => WriteOne(record, catalogEntry, holdsIds)); + } + + /// + /// Writes one typed document to the store file without a cache, as a cache write would. Element ids are decided before anything + /// is written: an unset id is minted, a list a cache write refuses refuses the write, and the file is marked as holding element + /// ids when the document holds one. A write that fails gives the minted ids back. Then the document is encoded as + /// , the type its schema id and ids are taken from, with , and stored as + /// stores a record: every other record and catalog entry is copied forward as stored, under the store's + /// lock, and the file is replaced atomically. + /// + /// The registered document type. A derived instance is stored as this type. + /// The record key the document is stored under. + /// The document to store. + /// The stored-at stamp, or null for now. + /// The registry that describes and encodes the document, or null for . The document type's entry is taken from it and is the only description of that schema id. Its other types claim stored records as a cache registering them would, and it declares nothing, so it can only refuse more: the document's entry owns a record whose id it declares (its own or one it lists compatible), and a record it reads whole when it is the picked claimant; refused are a record another registered type claims, one whose stored entry lists another registered type's id ahead of the entry's, and one that two registered types list or two versions of the schema could read. A wider registry (Shared when none is passed) refuses at least as much. It serves this call only: the store keeps no registry. + /// The call holds the store's gate while it runs. User code it runs (property getters while the document is encoded, constructors while a copied record is read) must not enter a cache or another store; that throws , carried by the encoder's wrapper when the code ran inside it. + /// The store is attached to a cache; write through the cache. + /// The file at the path is not a store this runtime can read. + /// The document holds an element id or list a cache write refuses. + public void WriteDocument(CultRecordKey key, TDocument document, string? storedAt = null, CultDocumentRegistry? registry = null) + where TDocument : class + { + if (document == null) throw new ArgumentNullException(nameof(document)); + var resolvedRegistry = registry ?? CultDocumentRegistry.Shared; + var catalogEntry = resolvedRegistry.GetRequired().ToCatalogEntry(); + Cacheless(resolvedRegistry, () => + { + var ids = CultElementIds.Plan(typeof(TDocument), document, key.Value, deterministic: false); + ids.Apply(); + try + { + WriteOne( + new CultPersistedRecord + { + Key = key.Value, + SchemaId = catalogEntry.SchemaId, + StoredAt = string.IsNullOrWhiteSpace(storedAt) ? DateTimeOffset.UtcNow.ToString("O") : storedAt!, + Payload = SerializePayload(document, typeof(TDocument)) + }, + catalogEntry, + CultElementIds.Holds(typeof(TDocument), document)); + } + catch + { + ids.Undo(); + throw; + } + }); + } + + private void WriteOne(CultPersistedRecord record, CultSchemaCatalogEntry catalogEntry, bool? holdsIds) + { + using (AcquireLock(wait: true)) + _ = ApplyWriteSet(ReadSnapshot(), new[] { new CultStagedRecord(record, catalogEntry, holdsIds) }, Array.Empty(), Array.Empty()); + } + + // Flush, commit and Mesh's write all end here, with the durable store they read under the lock they hold. First, a key the write + // names or checks whose durable record is under an id the writer does not declare refuses the write (RefuseUndeclaredOnDisk); a + // write that then has nothing to write but keys it checked writes no file. Then a store that holds + // a variant, or would after this write, is written only while its record versions are what this store last read: a variant + // resolves against other records, and a write decodes none it did not stage, so it cannot judge another writer's. Versions + // only: a write hashes other records' bytes, never decodes them. That holds for the variant gate; the one decode of a copied + // record is the element-id check (CopiedHoldIds), which runs only on an unmarked store and can only mark it. Otherwise the whole write is refused and what it refused is + // reloaded (Refuse). Every record the writer staged is encoded from its object now, and its version is the hash of that + // encoding; every other record is copied as the slice the file holds. Returns the records written, each with the version of + // the bytes written. + private IReadOnlyCollection<(CultStagedRecord Staged, string Version)> ApplyWriteSet( + Durable? durable, IReadOnlyCollection upserts, IReadOnlyCollection removals, IEnumerable checks) + { + RefuseUndeclaredOnDisk(durable, upserts, removals, checks); + if (upserts.Count == 0 && removals.Count == 0 && (checks.Any() || StagedChecks.Count > 0)) + return Array.Empty<(CultStagedRecord, string)>(); + RefuseWriteIntoMovedVariantStore(durable, upserts, removals); + var durableRecords = durable?.Snapshot.Records ?? Array.Empty(); + // A record written over one the store holds is stored at a storedAt later than the one it replaces, whoever wrote that. + var replaced = durableRecords.ToDictionary(record => record.Key, record => record.StoredAt, StringComparer.Ordinal); + foreach (var staged in upserts) + { + if (replaced.TryGetValue(staged.Record.Key, out var durableAt)) + staged.Record.StoredAt = StoredAtOver(staged.Record.StoredAt, durableAt); + } + + var written = upserts.Select(staged => staged.Record.Key).ToHashSet(StringComparer.Ordinal); + var removed = removals.ToHashSet(StringComparer.Ordinal); + var copied = durableRecords + .Where(record => !written.Contains(record.Key) && !removed.Contains(record.Key)) + .ToArray(); + var encoded = upserts.Select(staged => (Staged: staged, Bytes: SerializeRecord(staged.Record))).ToArray(); + var records = copied.Concat(upserts.Select(staged => staged.Record)).ToArray(); + var fileRecords = copied.Select(record => (record.Key, Bytes: durable!.Slice(record.Key))) + .Concat(encoded.Select(pair => (pair.Staged.Record.Key, pair.Bytes))) + .OrderBy(pair => pair.Key, StringComparer.Ordinal) + .Select(pair => pair.Bytes) + .ToArray(); + var catalog = CultSchemaCatalogEntry.Derive( + records, + upserts.Select(staged => staged.Entry).Distinct().ToArray(), + durable?.Snapshot.SchemaCatalog ?? Array.Empty()) + .OrderBy(entry => entry.SchemaName, StringComparer.Ordinal) + .ThenBy(entry => entry.SchemaId, StringComparer.Ordinal) + .Select(entry => durable?.CatalogBytes(entry) ?? SerializeCatalogEntry(entry)) + .ToArray(); + var format = HeaderForWrite( + durable?.Snapshot.FormatVersion, + upserts.Select(staged => staged.HoldsIds).ToArray(), + copiesAny: copied.Length > 0, + copiedHoldIds: CopiedHoldIds(durable, copied, upserts), + directoryStore: false, + holdsVariants: records.Any(record => record.Variant != null)); + + var versions = encoded.Select(pair => (pair.Staged, VersionOfBytes(pair.Bytes))).ToArray(); + Directory.CreateDirectory(FileInfo.DirectoryName!); + WriteSnapshotAtomically(FileInfo.FullName, SerializeStore(format, catalog, fileRecords)); + foreach (var key in removed) + _lastRead.Remove(key); + foreach (var (staged, version) in versions) + _lastRead[staged.Record.Key] = version; + return versions; + } + + // Whether a record this write copies forward holds an element id, decided from the bytes the file holds under the store's + // lock, so every C# writer with a type for the record decides the same from the same bytes. Positive-only: it runs only + // while the header on disk is unmarked and can only mark the store, so a narrower type that decodes no ids changes nothing. + // It never refuses a write: a record that does not resolve or decode is copied as its bytes and decides nothing. Gates, in + // order, so the decode runs only where it can change the answer. + private bool CopiedHoldIds(Durable? durable, CultPersistedRecord[] copied, IReadOnlyCollection upserts) + { + if (durable == null || + string.Equals(durable.Snapshot.FormatVersion, CultPersistedStoreSnapshot.FormatV3, StringComparison.Ordinal) || + upserts.Any(staged => staged.HoldsIds == true) || + AttachedRegistry is not { } registry) + return false; + var catalog = durable.Snapshot.SchemaCatalog; + foreach (var record in copied) + { + if (record.Variant != null) + continue; + try + { + if (registry.Claimant(record.SchemaId, catalog, out _) == null) + continue; + var type = registry.ResolvePersistedSchemaDetailed(record.SchemaId, catalog).Descriptor.DocumentType; + if (!CultElementIds.CanHoldIds(type)) + continue; + CopiedRecordsDecoded++; + if (CultElementIds.Holds(type, DeserializePayload(type, record.Payload))) + return true; + } + catch (Exception e) when (e is not OperationCanceledException) + { + // Copied as its bytes: no type, or no decode, is no fact. + } + } + + return false; + } + + private void RefuseWriteIntoMovedVariantStore(Durable? durable, IReadOnlyCollection upserts, IReadOnlyCollection removals) + { + var records = durable?.Snapshot.Records ?? Array.Empty(); + if (records.All(record => record.Variant == null) && upserts.All(staged => staged.Record.Variant == null)) return; - try + var changed = new HashSet(StringComparer.Ordinal); + var versions = records.ToDictionary(record => record.Key, record => durable!.Version(record.Key), StringComparer.Ordinal); + foreach (var (key, version) in versions) { - DeserializeSnapshot(bytes); + if (!_lastRead.TryGetValue(key, out var read) || !SameStoredIdentity(read, version)) + changed.Add(key); } - catch (NotSupportedException) - { - throw; - } - catch (Exception) + + var durableKeys = records.Select(record => record.Key).ToHashSet(StringComparer.Ordinal); + foreach (var key in _lastRead.Keys) { + if (!durableKeys.Contains(key)) + changed.Add(key); } + + if (changed.Count == 0) + return; + var recordKeys = _staged.Concat(upserts.Select(staged => staged.Record.Key)).Concat(removals) + .Distinct(StringComparer.Ordinal).OrderBy(key => key, StringComparer.Ordinal).ToArray(); + var changedKeys = changed.OrderBy(key => key, StringComparer.Ordinal).ToArray(); + Refuse( + recordKeys.Concat(changedKeys), + key => Cache == null ? null : DurableAt(durable, key), + key => ForeignAt(durable, key), + key => + { + _staged.Remove(key); + IsDirty = _staged.Count > 0; + if (versions.TryGetValue(key, out var version)) + _lastRead[key] = version; + else + _lastRead.Remove(key); + }, + inner => new CultWriteConflictException( + Cache == null + ? $"{this} holds a variant and was written by a writer with no CultCache, which cannot resolve variants; nothing was written. " + + $"Records it would have written or removed: {string.Join(", ", recordKeys.Select(key => $"'{key}'"))}; records that differ from what the writer has read: {string.Join(", ", changedKeys.Select(key => $"'{key}'"))}." + : $"{this} changed since this cache last read it, and it holds a variant, which resolves against other records; nothing was written. " + + $"Records this write would have written or removed: {string.Join(", ", recordKeys.Select(key => $"'{key}'"))}; records that changed: {string.Join(", ", changedKeys.Select(key => $"'{key}'"))}. " + + (inner == null + ? "The cache reloaded them; retry the write against what the store holds now." + : "Reloading them failed, so the cache is unchanged."), + recordKeys, + changedKeys, + inner)); } - private byte[]? ReadFileBytes() + // A writable store judges a removal of a key its cache does not hold at its next flush or commit, under the lock (StageChecks); + // a read-only store has neither, so it answers from its view and never creates the lock file. + protected internal override void RefuseRemovalOfUndeclared(IReadOnlyCollection keys) { - FileInfo.Refresh(); - if (!FileInfo.Exists) + if (IsReadOnly) + base.RefuseRemovalOfUndeclared(keys); + else + StageChecks(keys); + } + + // A read-only store answers a batch's checks from its view and never creates the lock file; a writable one reads the file + // under its lock. + protected internal override CultCommitOutcome JudgeChecks(IReadOnlyCollection keys, bool wait) + { + if (IsReadOnly) + return base.JudgeChecks(keys, wait); + return Held(() => + { + using var fileLock = AcquireLock(wait); + if (fileLock == null) + return CultCommitOutcome.Contended; + RefuseUndeclaredOnDisk(ReadSnapshot(), keys, OwnsOnDisk()); + return CultCommitOutcome.Committed; + }); + } + + // A cache owns through its registry (CultDocumentRegistry.Owns). A cacheless CultMesh write owns what the catalog entries it + // writes declare: their own ids and the ids they list compatible, or an id whose publishing entry its claimant reads whole, + // the claimant being picked by the one precedence over the written entries and the lent registry's other types, and owning + // only when it is a written entry (variants:ruling:mesh-writes-bound-by-declaration). The keys judged are every key staged, + // written, removed or checked. + private void RefuseUndeclaredOnDisk(Durable? durable, IReadOnlyCollection upserts, IReadOnlyCollection removals, IEnumerable checks) => + RefuseUndeclaredOnDisk( + durable, + _staged.Concat(upserts.Select(staged => staged.Record.Key)).Concat(removals).Concat(checks).Concat(StagedChecks), + OwnsOnDisk(upserts)); + + // The owns predicate a write over the durable catalog asks. A cacheless writer declares what the entries it writes declare + // (variants:ruling:mesh-writes-bound-by-declaration) and reads a published entry with one lookup: the written entries shadow + // the lent registry's type for their ids, and the registry adds other claimants. The record is owned only when the claimant + // is a written entry, so the call is never more permissive than that declaration, and a registry can only refuse more. + private Func, bool> OwnsOnDisk(IReadOnlyCollection? upserts = null) => + Cache != null || upserts == null + ? Registry.Owns + : (id, catalog) => + { + var writes = upserts.Select(staged => staged.Entry).ToArray(); + Func declared = candidate => writes.Any(entry => + string.Equals(entry.SchemaId, candidate, StringComparison.Ordinal) || + entry.CompatibleSchemaIds.Contains(candidate, StringComparer.Ordinal)); + return CultDocumentRegistry.Owns( + id, + catalog, + declared, + new CatalogEntryLookup(writes, AttachedRegistry), + entry => entry, + claimant => writes.Any(entry => string.Equals(entry.SchemaId, claimant.SchemaId, StringComparison.Ordinal))); + }; + + // A refused key's durable version is what this store has now read of it. + private void RefuseUndeclaredOnDisk(Durable? durable, IEnumerable written, Func, bool> owns) + { + var undeclared = UndeclaredOnDisk(durable, owns); + if (undeclared.Count == 0) + return; + RefuseUndeclaredOnDisk( + written, + key => undeclared.TryGetValue(key, out var carried) ? carried : null, + key => Cache == null ? null : DurableAt(durable, key), + key => ForeignAt(durable, key), + key => + { + _staged.Remove(key); + IsDirty = _staged.Count > 0; + _lastRead[key] = durable!.Version(key); + }, + cacheless: Cache == null); + } + + // Every durable record under an id the writer does not own, by key, named by the catalog entry that publishes its id. The + // answer is decided once per distinct schema id, so a large store pays one comparison per id, not per record. + private static Dictionary UndeclaredOnDisk(Durable? durable, Func, bool> owns) + { + var undeclared = new Dictionary(StringComparer.Ordinal); + var answers = new Dictionary(StringComparer.Ordinal); + bool OwnsOnce(string id, IReadOnlyCollection catalog) => + answers.TryGetValue(id, out var answer) ? answer : answers[id] = owns(id, catalog); + foreach (var record in durable?.Snapshot.Records ?? Array.Empty()) + { + if (Undeclared(record, durable!.Snapshot.SchemaCatalog, OwnsOnce) is { } carried) + undeclared[record.Key] = carried; + } + + return undeclared; + } + + // The durable record at a key as a load lists it when no registered type claims its id; null otherwise. + private CultForeignRecord? ForeignAt(Durable? durable, string key) + { + if (durable?.Snapshot.Records.FirstOrDefault(candidate => string.Equals(candidate.Key, key, StringComparison.Ordinal)) is not { } record || + Foreign(record, durable.Snapshot.SchemaCatalog, durable.Version(key)) is not { } carried) + return null; + return carried; + } + + // The durable record at a key decoded as a load decodes it, at its durable version: null when the store holds none there or + // holds it foreign. The one decode both refusals reload through. + private CultStoredDocument? DurableAt(Durable? durable, string key) + { + var record = durable?.Snapshot.Records.FirstOrDefault(candidate => string.Equals(candidate.Key, key, StringComparison.Ordinal)); + if (record == null || Foreign(record, durable!.Snapshot.SchemaCatalog, durable.Version(key)) != null) + return null; + return ToStoredDocument(record, durable.Version(key), durable.Snapshot.SchemaCatalog, DeserializePayload, out _); + } + + // The file now holds every record written, under the id and at the storedAt the write gave it, and the version of those bytes: + // the store's entries and the cache's say so. A copied record is on disk as the file had it, so nothing learns anything new about it: in particular the + // ids it minted at load are still in memory only. + private void Wrote(IEnumerable<(CultStagedRecord Staged, string Version)> written) + { + var staged = written.ToArray(); + foreach (var (write, version) in staged) + { + if (!Entries.TryGetValue(write.Record.Key, out var entry)) + continue; + entry.StoredSchemaId = write.Record.SchemaId; + entry.StoredAt = write.Record.StoredAt; + entry.StoredVersion = version; + } + + Cache?.Wrote(staged.Select(write => (write.Staged.Record, write.Version))); + } + + // Open, flush and commit read the file through this one call, so they agree on which files are stores. A file that is + // gone is an empty store: nothing to read, nothing to lose. A zero-byte file is not a store; it is refused. + private Durable? ReadSnapshot() + { + var bytes = ReadDisk(); + if (bytes == null) return null; try + { + return new Durable(this, ReadStore(bytes)); + } + catch (CultStoreUnreadableException ex) when (ex.Path == null) + { + throw new CultStoreUnreadableException(ex.Message, FileInfo.FullName, ex.InnerException); + } + } + + /// + /// Reads the store file once, under the store's gate, with the reader a cache's store opens with, and returns what is + /// durable at the path: the records, the schema catalog and the format stamp. Returns null when nothing is at the path. + /// CultMesh's single-file document reads are built on this, so they share the store's one reader and its stamp rule. + /// + /// The file is empty, truncated, or stamped with a format this runtime does not read. + public CultPersistedStoreSnapshot? ReadStoreSnapshot() => Held(() => ReadSnapshot()?.Snapshot); + + private byte[]? ReadDisk() + { + FileInfo.Refresh(); + // Only nothing at the path is an empty store: a path with nothing at it reports no attributes at all (-1). A link whose + // target is gone is a store that cannot be reached, and a directory is not a store file. + var attributes = FileInfo.Attributes; + if ((int)attributes == -1) + return null; + if ((attributes & FileAttributes.Directory) != 0 && (attributes & FileAttributes.ReparsePoint) == 0) + throw new IOException("The store path is a directory, not a store file."); + try { return ReadAllBytesShared(FileInfo.FullName); } + catch (FileNotFoundException ex) when (PathIsALink()) + { + throw new IOException("The store path is a link whose target cannot be reached.", ex); + } + // A Windows junction is a directory, not a file, so opening one that dangles fails with an access error, not a missing file. + catch (UnauthorizedAccessException ex) when (PathIsALink() && (FileInfo.Attributes & FileAttributes.Directory) != 0) + { + throw new IOException("The store path is a link whose target cannot be reached.", ex); + } catch (FileNotFoundException) { return null; } } - private CultPersistedStoreSnapshot? ReadSnapshot() + // Something is at the path even when the path cannot be opened: a link keeps the reparse-point attribute whether or not its + // target is there. A path with nothing at it reports no attributes at all (-1). + private bool PathIsALink() { - var bytes = ReadFileBytes(); - return bytes == null ? null : DeserializeSnapshot(bytes); - } - - private void WriteSnapshot(IEnumerable records, IEnumerable catalog) - { - var ordered = records.OrderBy(record => record.Key, StringComparer.Ordinal).ToArray(); - var used = new HashSet(ordered.Select(record => record.SchemaId), StringComparer.Ordinal); - var snapshot = new CultPersistedStoreSnapshot - { - // v2 exactly when the store holds a variant: a store without one stays byte-identical v1. - FormatVersion = ordered.Any(record => record.Variant != null) - ? CultPersistedStoreSnapshot.FormatV2 - : CultPersistedStoreSnapshot.FormatV1, - SchemaCatalog = catalog - .GroupBy(entry => entry.SchemaId, StringComparer.Ordinal) - .Select(group => group.First()) - .Where(entry => used.Contains(entry.SchemaId)) - .OrderBy(entry => entry.SchemaName, StringComparer.Ordinal) - .ToArray(), - Records = ordered - }; - - Directory.CreateDirectory(FileInfo.DirectoryName!); - WriteSnapshotAtomically(FileInfo.FullName, SerializeSnapshot(snapshot)); + FileInfo.Refresh(); + var attributes = FileInfo.Attributes; + return (int)attributes != -1 && (attributes & FileAttributes.ReparsePoint) != 0; } // The lock is a sidecar opened exclusively, which excludes other handles in this process and in others alike. diff --git a/src/GameCult.Caching/CultDocumentContracts.cs b/src/GameCult.Caching/CultDocumentContracts.cs index 442f5c4f..b439ac9f 100644 --- a/src/GameCult.Caching/CultDocumentContracts.cs +++ b/src/GameCult.Caching/CultDocumentContracts.cs @@ -18,6 +18,14 @@ namespace GameCult.Caching public string SchemaName { get; } public string SchemaVersion { get; } + + /// + /// Schema ids other than this type's own that a store may hold its records under, typically the ids an earlier + /// version of the schema had. The registered catalog entry lists them, and a cache resolves a record stored under + /// one of them to this type (its own id is tried first) and restamps it to the type's own id on load. TypeScript + /// and Python name the same declaration compatibleSchemaIds. + /// + public string[] CompatibleSchemaIds { get; set; } = Array.Empty(); } [AttributeUsage(AttributeTargets.Class, Inherited = false)] diff --git a/src/GameCult.Caching/CultElementIds.cs b/src/GameCult.Caching/CultElementIds.cs new file mode 100644 index 00000000..38dfaaae --- /dev/null +++ b/src/GameCult.Caching/CultElementIds.cs @@ -0,0 +1,684 @@ +using System; +using System.Collections; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Reflection; +using System.Runtime.Serialization; +using System.Security.Cryptography; +using System.Text; +using MessagePack; + +namespace GameCult.Caching +{ + // The identity of one element of an object list. A string member, keyed like any other, on every object type that + // sits in a list of a registered document type (union bases included: every concrete subtype carries the member). + // [CultElementId(nameof(Offset))] declares the id derived, never random: an unset id becomes the invariant-culture text of + // that member (a keyed string or number on the same type, unique within its list). Content-addressed elements use it so the + // same content always encodes to the same bytes. + [AttributeUsage(AttributeTargets.Field | AttributeTargets.Property, Inherited = true, AllowMultiple = false)] + public sealed class CultElementIdAttribute : Attribute + { + public CultElementIdAttribute(string? derivedFrom = null) + { + DerivedFrom = derivedFrom; + } + + public string? DerivedFrom { get; } + } + + // A record the id rule refuses: a duplicate id in one list, a random id that is not 12 lowercase hex characters, or a derived + // id whose source member is null or empty. It names the record, the list (record key, member slot, dictionary key and + // element index) and, where there is one, the element id and the source member. + public sealed class CultElementIdException : InvalidOperationException + { + public CultElementIdException(string message, string recordKey, string listPath, string? elementId = null, string? member = null) + : base(message) + { + RecordKey = recordKey; + ListPath = listPath; + ElementId = elementId; + Member = member; + } + + public string RecordKey { get; } + public string ListPath { get; } + public string? ElementId { get; } + public string? Member { get; } + } + + // The one owner of element identity: which types need an id, and how unset ids are filled. + // A random id is 12 lowercase hex characters (48 bits); a derived id is the invariant text of its source and only has to be + // non-empty. Either need only be unique within one list. A write mints a random id; a load of a store written before ids + // existed mints from SHA-256(record key, element path), so every reader of the same store mints the same id and a reload is + // identical to what was read. A write and a load refuse the same things. + internal static class CultElementIds + { + private const int IdBytes = 6; + + /// The element ids a pass would fill, decided before anything changes. sets them; gives back what Apply replaced. + internal sealed class IdPlan + { + private readonly List<(object Item, Shape Shape, string? Before, string Id)> _fills = new(); + + public int Count => _fills.Count; + + internal void Add(object item, Shape shape, string? before, string id) => _fills.Add((item, shape, before, id)); + + public void Apply() + { + foreach (var (item, shape, _, id) in _fills) + shape.SetId!(item, id); + } + + public void Undo() + { + foreach (var (item, shape, before, _) in _fills) + shape.SetId!(item, before!); + } + } + + internal sealed class Shape + { + public Func? GetId; + public Action? SetId; + public Func? Derive; + public string? DerivedFrom; + public (int Slot, Type Type, Func Get)[] Walk = Array.Empty<(int, Type, Func)>(); + } + + private static readonly ConcurrentDictionary Shapes = new(); + + // ---- registration: which element types must carry an id ---- + + internal sealed class Survey + { + public List Problems { get; } = new(); + + // Every type whose bytes carry an element id: the types a walk descends through to reach an object list, the list's + // element types and every concrete type under them. A serialization codec must write all of these by their [Key] + // layout, so it checks them against the formatters it composed. + public List IdPath { get; } = new(); + + // The concrete element types already judged in this document, so each is refused at most once however many lists hold it. + internal HashSet Judged { get; } = new(); + } + + // What the walk saw, settled once it is complete: a type reaches a list only through types reached later, and a type + // can sit in a cycle, so the path is a fixed point over these edges and not a stack. + private sealed class Graph + { + public List<(Type Owner, MemberInfo? Member, Type Child, Type[] Through)> Edges { get; } = new(); + public List<(Type Owner, MemberInfo Member, Type Element, List Concrete, Type[] Through)> Lists { get; } = new(); + + // Keyed types that also enumerate objects: the walk reads them by key, so the codec must confirm MessagePack does too. + public List Enumerating { get; } = new(); + } + + private static readonly ConcurrentDictionary IdCapable = new(); + + // A document type whose walk reaches no object list can hold no element id, so a write never decodes one to ask. + internal static bool CanHoldIds(Type documentType) => IdCapable.GetOrAdd(documentType, type => Inspect(type).IdPath.Count > 0); + + // Every reason this document type cannot be registered under the id rule, one per offending concrete element type. + internal static Survey Inspect(Type documentType) + { + var survey = new Survey(); + var graph = new Graph(); + VisitObject(documentType, documentType, new HashSet(), survey, graph); + Settle(documentType, survey, graph); + return survey; + } + + private static void VisitObject(Type documentType, Type type, HashSet seen, Survey survey, Graph graph) + { + if (!seen.Add(type)) + return; + if (EnumeratesObjects(type)) + graph.Enumerating.Add(type); + foreach (var (member, memberType) in KeyedMembers(type)) + VisitMemberType(documentType, type, member, memberType, Array.Empty(), seen, survey, graph); + foreach (var union in type.GetCustomAttributes(true)) + { + graph.Edges.Add((type, null, union.SubType, Array.Empty())); + VisitObject(documentType, union.SubType, seen, survey, graph); + } + } + + private static void VisitMemberType(Type documentType, Type owner, MemberInfo member, Type memberType, Type[] through, HashSet seen, Survey survey, Graph graph) + { + memberType = Nullable.GetUnderlyingType(memberType) ?? memberType; + // A sequence or map whose element is itself (an unkeyed Node : IEnumerable) ends the type walk here; without + // this it would recurse until the stack overflows. + if (Array.IndexOf(through, memberType) >= 0) + return; + var (kind, inner) = Classify(memberType); + switch (kind) + { + case Kind.Leaf: + // A shape the walk does not own is one whole value, never refused (q3 amended). + return; + case Kind.Map: + VisitMemberType(documentType, owner, member, inner, through.Append(memberType).ToArray(), seen, survey, graph); + return; + case Kind.Sequence when IsElement(inner): + var concretes = ConcreteTypes(inner).ToList(); + graph.Lists.Add((owner, member, inner, concretes, through.Append(memberType).ToArray())); + foreach (var concrete in concretes) + { + if (!survey.Judged.Add(concrete)) + continue; + var problem = IsElement(concrete) ? IdProblem(concrete) : UnkeyedElementProblem; + if (problem != null) + survey.Problems.Add($"Cult document {documentType.Name}: list element type {concrete.Name} (in {owner.Name}.{member.Name}) {problem}"); + } + + VisitObject(documentType, inner, seen, survey, graph); + return; + case Kind.Sequence: + VisitMemberType(documentType, owner, member, inner, through.Append(memberType).ToArray(), seen, survey, graph); + return; + default: + graph.Edges.Add((owner, member, memberType, through)); + VisitObject(documentType, memberType, seen, survey, graph); + return; + } + } + + private const string UnkeyedElementProblem = + "is written by a formatter the cache does not own, so its element id would never reach disk; mark it [MessagePackObject] with [Key] members, or keep it out of lists."; + + // The id path: every type that reaches an object list through members MessagePack writes by key, the lists' element and + // concrete types, and the keyed types that enumerate objects. The codec verifies that MessagePack writes each of them the + // way Classify reads it; it never decides ownership. + private static void Settle(Type documentType, Survey survey, Graph graph) + { + var lists = graph.Lists; + var reaching = new HashSet(lists.Select(list => list.Owner)); + var through = new List(lists.SelectMany(list => list.Through)); + for (var grew = true; grew;) + { + grew = false; + foreach (var (owner, _, child, via) in graph.Edges) + { + if (!reaching.Contains(child)) + continue; + through.AddRange(via); + grew |= reaching.Add(owner); + } + } + + var path = new List(); + void Add(Type type) + { + if (!path.Contains(type)) + path.Add(type); + } + + foreach (var type in reaching) + Add(type); + foreach (var type in through) + Add(type); + foreach (var list in lists) + { + Add(list.Element); + foreach (var concrete in list.Concrete) + Add(concrete); + } + + // A keyed type the walk reads by key but whose enumeration holds objects is on the path: the codec asks whether + // MessagePack writes it by key too. + foreach (var type in graph.Enumerating) + Add(type); + + survey.IdPath.AddRange(path); + } + + // The types an element can actually be: the element type itself when it can be instantiated, and every union subtype + // under it. Shapes are built from these, so registration judges exactly what minting will meet. + private static IEnumerable ConcreteTypes(Type element) + { + var seen = new HashSet(); + return Expand(element).ToList(); + + IEnumerable Expand(Type type) + { + if (!seen.Add(type)) + yield break; + if (!type.IsAbstract && !type.IsInterface) + yield return type; + foreach (var union in type.GetCustomAttributes(true)) + { + foreach (var subtype in Expand(union.SubType)) + yield return subtype; + } + } + } + + private static string? IdProblem(Type element) + { + // Candidates come from every instance member up the base chain, public or not, so an id MessagePack does not write + // is named as such instead of being reported missing. An override or a hiding member is one member, the most derived. + var ids = AllInstanceMembers(element).Where(member => Declares(member, typeof(CultElementIdAttribute))) + .GroupBy(member => member.Name).Select(group => group.First()).ToArray(); + if (ids.Length == 0) + return "has no [CultElementId] member; add a public string member with [Key] and [CultElementId] so each element has an identity."; + if (ids.Length > 1) + return $"has two [CultElementId] members ({ids[0].Name}, {ids[1].Name}); an element has one identity."; + var id = ids[0]; + if (Declares(id, typeof(MessagePackFormatterAttribute))) + return $"declares [MessagePackFormatter] on its [CultElementId] member {id.Name}, so the id would not be written by its [Key]."; + if (TypeOf(id) != typeof(string)) + return $"declares [CultElementId] on {id.Name}, which is not a string."; + if (KeyOf(id) is not { } idKey) + return $"declares [CultElementId] on {id.Name}, which has no integer [Key], so it is never persisted."; + if (MembersOf(element).FirstOrDefault(member => member.Name == id.Name && member.MemberType == id.MemberType) is not { } written) + return $"declares [CultElementId] on {id.Name}, which MessagePack does not write ([IgnoreMember], or non-public on a type without [MessagePackObject(AllowPrivate = true)]); the id would never reach disk."; + id = written; + var sharing = KeyedMembers(element).FirstOrDefault(entry => entry.Member != id && KeyOf(entry.Member) == idKey); + if (sharing.Member != null) + return $"declares [CultElementId] on {id.Name} with [Key({idKey})], which {sharing.Member.Name} also uses; an id needs a key of its own."; + if (id is PropertyInfo { SetMethod: null } or FieldInfo { IsInitOnly: true }) + return $"declares [CultElementId] on {id.Name}, which cannot be assigned; the cache mints into it."; + if (id is PropertyInfo { SetMethod.IsPublic: false } && !AllowsPrivate(element)) + return $"declares [CultElementId] on {id.Name}, whose setter is not public: MessagePack writes the id but does not read it back, so every load would mint a different one. Make the setter public (init is fine)."; + var source = id.GetCustomAttribute(true)!.DerivedFrom; + if (source != null) + { + var from = MembersOf(element).FirstOrDefault(member => member.Name == source); + if (from == null || KeyOf(from) == null) + return $"derives its [CultElementId] from {source}, which is not a keyed member of the type."; + var kind = Nullable.GetUnderlyingType(TypeOf(from)) ?? TypeOf(from); + if (kind != typeof(string) && !kind.IsPrimitive) + return $"derives its [CultElementId] from {source}, which is not a string or a number."; + } + + return null; + } + + // ---- write: what the stored form holds ---- + + // True when some element under root holds a non-empty id: the one fact a store marks its header by. It reads the value + // as it is now, refuses nothing and changes nothing, so a writer asks it of exactly what it is about to write. The walk + // follows declared types, as registration does; a value's own type picks only a union arm. + /// True when some element under holds a non-empty id: the one fact a single-file store marks its header by. Reads the value as it is, refuses nothing and changes nothing. + /// The declared type of the root; the walk follows declared types, as registration does. + /// The document about to be written. + public static bool Holds(Type declared, object? root) + { + if (root == null) + return false; + var (kind, inner) = Classify(declared); + switch (kind) + { + case Kind.Leaf: + return false; + case Kind.Map: + return MapEntries(declared)(root).Any(entry => Holds(inner, entry.Value)); + case Kind.Sequence: + var elements = IsElement(inner); + if (!elements && Classify(inner).Kind == Kind.Leaf) + return false; + foreach (var item in (IEnumerable)root) + { + if (item == null) + continue; + if (elements && OwnerShape(inner, item).GetId is { } getId && !string.IsNullOrEmpty(getId(item))) + return true; + if (Holds(inner, item)) + return true; + } + + return false; + default: + return OwnerShape(declared, root).Walk.Any(member => Holds(member.Type, member.Get(root))); + } + } + + // ---- write and load: mint and check ---- + + // Decides every unset element id under root, and refuses what the rule refuses, without changing anything: a random id + // that is not 12 lowercase hex characters, a duplicate id in one list, a derived id whose source is null or empty. + // rootPath (the record key) seeds deterministic minting and names the record in a refusal. An element object that sits + // twice in one list is refused too: planning would give it two ids and it would then be a duplicate on load. + /// Decides every unset element id under and refuses what a cache write refuses (a malformed or duplicate id, an unusable derived source), without changing anything until . A cacheless single-file writer plans before it serializes, so the ids are in the bytes. + /// The declared type of the root. + /// The document being written or loaded. + /// The record key: it seeds deterministic minting and names the record in a refusal. + /// True on a load, which mints from the key and path so every reader of one store mints the same ids; false on a write, which mints random ones. + public static IdPlan Plan(Type declared, object? root, string rootPath, bool deterministic) + { + var plan = new IdPlan(); + Visit(declared, root, rootPath); + return plan; + + void Visit(Type type, object? value, string path) + { + if (value == null) + return; + var (kind, inner) = Classify(type); + switch (kind) + { + case Kind.Map: + foreach (var entry in MapEntries(type)(value)) + Visit(inner, entry.Value, path + "{" + Convert.ToString(entry.Key, CultureInfo.InvariantCulture) + "}"); + break; + case Kind.Sequence when IsElement(inner): + Elements(inner, (IEnumerable)value, path); + break; + case Kind.Sequence: + if (Classify(inner).Kind == Kind.Leaf) + break; + var index = 0; + foreach (var item in (IEnumerable)value) + Visit(inner, item, path + "/" + (index++).ToString(CultureInfo.InvariantCulture)); + break; + case Kind.Owner: + foreach (var (slot, memberType, get) in OwnerShape(type, value).Walk) + Visit(memberType, get(value), path + "." + slot.ToString(CultureInfo.InvariantCulture)); + break; + } + } + + void Elements(Type element, IEnumerable list, string path) + { + var items = list.Cast().ToArray(); + var taken = new HashSet(StringComparer.Ordinal); + var objects = new HashSet(ReferenceComparer.Instance); + foreach (var item in items) + { + if (item == null || OwnerShape(element, item) is not { GetId: { } getId } shape) + continue; + if (!objects.Add(item)) + throw new CultElementIdException( + $"Record {rootPath}: the same {item.GetType().Name} object appears twice in the list at {path}; an element is one object with one id.", + rootPath, path); + var id = getId(item); + if (string.IsNullOrEmpty(id)) + continue; + if (shape.Derive == null && !IsRandomId(id!)) + throw new CultElementIdException( + $"Record {rootPath}: element id '{id}' in the list at {path} is not 12 lowercase hex characters. An id you write by hand must be 12 lowercase hex; for readable ids, derive them from a member with [CultElementId(nameof(Member))].", + rootPath, path, id); + if (!taken.Add(id!)) + throw Duplicate(id!, path); + } + + for (var index = 0; index < items.Length; index++) + { + var item = items[index]; + if (item == null) + continue; + var itemPath = path + "/" + index.ToString(CultureInfo.InvariantCulture); + if (OwnerShape(element, item) is { SetId: { } , GetId: { } get } itemShape && + string.IsNullOrEmpty(get(item))) + { + string id; + if (itemShape.Derive != null) + { + id = itemShape.Derive(item) ?? string.Empty; + if (id.Length == 0) + throw new CultElementIdException( + $"Record {rootPath}: the element at {itemPath} derives its id from {itemShape.DerivedFrom}, which is null or empty.", + rootPath, itemPath, member: itemShape.DerivedFrom); + if (!taken.Add(id)) + throw Duplicate(id, path); + } + else + id = Mint(itemPath, deterministic, rootPath, taken); + plan.Add(item, itemShape, get(item), id); + } + + Visit(element, item, itemPath); + } + } + + CultElementIdException Duplicate(string id, string path) => new( + $"Record {rootPath}: element id '{id}' appears twice in the list at {path}; an element id is unique within its list.", + rootPath, path, id); + } + + private sealed class ReferenceComparer : IEqualityComparer + { + public static readonly ReferenceComparer Instance = new(); + + bool IEqualityComparer.Equals(object? left, object? right) => ReferenceEquals(left, right); + + int IEqualityComparer.GetHashCode(object value) => System.Runtime.CompilerServices.RuntimeHelpers.GetHashCode(value); + } + + private static bool IsRandomId(string id) + { + if (id.Length != IdBytes * 2) + return false; + foreach (var character in id) + { + if (!(character is >= '0' and <= '9' or >= 'a' and <= 'f')) + return false; + } + + return true; + } + + private static string Mint(string itemPath, bool deterministic, string rootPath, HashSet taken) + { + for (var attempt = 0; ; attempt++) + { + var bytes = new byte[IdBytes]; + if (deterministic) + { + using var sha = SHA256.Create(); + var hash = sha.ComputeHash(Encoding.UTF8.GetBytes(rootPath + "|" + itemPath + "|" + attempt.ToString(CultureInfo.InvariantCulture))); + Array.Copy(hash, bytes, IdBytes); + } + else + RandomBytes(bytes); + var id = Hex(bytes); + if (taken.Add(id)) + return id; + } + } + + private static void RandomBytes(byte[] bytes) + { + using var random = RandomNumberGenerator.Create(); + random.GetBytes(bytes); + } + + private static string Hex(byte[] bytes) + { + var text = new StringBuilder(bytes.Length * 2); + foreach (var value in bytes) + text.Append(value.ToString("x2", CultureInfo.InvariantCulture)); + return text.ToString(); + } + + // ---- shapes ---- + + private static Shape ShapeOf(Type type) => Shapes.GetOrAdd(type, Build); + + // The shape of an owner or element a walk holds under a declared type: a union base writes its arm, so the value's own + // type picks the arm; any other declared type is written by its own [Key] layout, whatever a subclass adds. + private static Shape OwnerShape(Type declared, object value) + { + declared = Nullable.GetUnderlyingType(declared) ?? declared; + return declared.IsDefined(typeof(UnionAttribute), false) ? ShapeOf(value.GetType()) : ShapeOf(declared); + } + + private static Shape Build(Type type) + { + var shape = new Shape(); + var idMember = MembersOf(type).FirstOrDefault(member => Declares(member, typeof(CultElementIdAttribute))); + if (idMember != null) + { + shape.GetId = idMember is FieldInfo field ? o => (string?)field.GetValue(o) : o => (string?)((PropertyInfo)idMember).GetValue(o); + shape.SetId = idMember is FieldInfo f ? (o, v) => f.SetValue(o, v) : (o, v) => ((PropertyInfo)idMember).SetValue(o, v); + if (idMember.GetCustomAttribute(true)!.DerivedFrom is { } source && + MembersOf(type).FirstOrDefault(member => member.Name == source) is { } from) + { + shape.DerivedFrom = source; + shape.Derive = o => Convert.ToString(from is FieldInfo ff ? ff.GetValue(o) : ((PropertyInfo)from).GetValue(o), CultureInfo.InvariantCulture); + } + } + + shape.Walk = KeyedMembers(type) + .Where(entry => Classify(entry.Type).Kind != Kind.Leaf) + .Select(entry => (Slot: KeyOf(entry.Member)!.Value, entry.Type, Get: entry.Member is FieldInfo fi + ? (Func)(o => fi.GetValue(o)) + : o => ((PropertyInfo)entry.Member).GetValue(o))) + .ToArray(); + return shape; + } + + // The members MessagePack's StandardResolver writes for an integer-keyed type, none of them [IgnoreMember]. Without + // AllowPrivate: public instance fields, and public instance properties with a public getter. With it, read from the most + // derived [MessagePackObject] declaration (a derived type without it is written without it, probe NoApDerived): every + // instance field and indexer-free property up the base chain at any visibility, the most derived of a name. + // TheWalkReadsTheMembersMessagePackWrites checks this set against MessagePack's own member discovery. + private static IEnumerable MembersOf(Type type) => + AllowsPrivate(type) + ? AllInstanceMembers(type).Where(member => !Declares(member, typeof(IgnoreMemberAttribute))) + .GroupBy(member => member.Name).Select(group => group.First()) + : PublicMembersOf(type); + + private static bool AllowsPrivate(Type type) => type.GetCustomAttribute(true)?.AllowPrivate == true; + + private static IEnumerable PublicMembersOf(Type type) => + type.GetFields(BindingFlags.Instance | BindingFlags.Public).Cast() + .Concat(type.GetProperties(BindingFlags.Instance | BindingFlags.Public) + .Where(property => property.GetIndexParameters().Length == 0 && property.GetMethod is { IsPublic: true })) + .Where(member => !Declares(member, typeof(IgnoreMemberAttribute))); + + // Every instance field and indexer-free property declared on the type or any base, most derived first. + private static IEnumerable AllInstanceMembers(Type type) + { + const BindingFlags flags = BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.DeclaredOnly; + for (var level = type; level != null && level != typeof(object); level = level.BaseType) + { + foreach (var field in level.GetFields(flags)) + yield return field; + foreach (var property in level.GetProperties(flags).Where(property => property.GetIndexParameters().Length == 0)) + yield return property; + } + } + + // The [Key] members MessagePack writes, for the oracle test that checks them against MessagePack. + internal static IEnumerable WrittenKeyedMembers(Type type) => MembersOf(type).Where(member => KeyOf(member) != null); + + private static IEnumerable<(MemberInfo Member, Type Type)> KeyedMembers(Type type) => + WrittenKeyedMembers(type).Where(member => !Declares(member, typeof(MessagePackFormatterAttribute))) + .Select(member => (member, TypeOf(member))); + + // An attribute a member declares, or inherits through an override: MemberInfo.IsDefined ignores inherit for properties, so an + // override of a base virtual member would not see its base's attribute. + private static bool Declares(MemberInfo member, Type attribute) => Attribute.IsDefined(member, attribute, inherit: true); + + // True when the walk reads the type by its [Key] members: the codec compares this with how MessagePack writes it. + internal static bool WalksAsOwner(Type type) => Classify(Nullable.GetUnderlyingType(type) ?? type).Kind == Kind.Owner; + + private static bool EnumeratesObjects(Type type) => + (MapTypes(type)?.Value ?? ElementTypeOf(type)) is { } inner && Classify(inner).Kind != Kind.Leaf; + + private static int? KeyOf(MemberInfo member) => member.GetCustomAttribute(true)?.IntKey; + + private static Type TypeOf(MemberInfo member) => member is FieldInfo field ? field.FieldType : ((PropertyInfo)member).PropertyType; + + // ---- type classification ---- + + private enum Kind { Leaf, Map, Sequence, Owner } + + private static readonly ConcurrentDictionary Classes = new(); + + // The walk's model of a declared type: registration, Plan, Holds and the codec's id path all go through it. A + // [MessagePackObject] or [Union] type is read by key even when it is also IEnumerable; the codec checks, for every type + // on the id path, that MessagePack writes it the way this model reads it, and refuses the document when it does not. + // Inner is a map's value type, a sequence's element type, or the type itself for an owner. + // The owned shapes are a closed list (Owner, Sequence, Map); every other shape is Leaf: one whole value with no element + // ids, replaced whole by a variant (q3 amended 2026-10-16). When in doubt the answer is Leaf. + // Leaf strings (an IEnumerable MessagePack writes whole), a type written whole (IsWrittenWhole: attribute- + // formatted, [DataContract], string-keyed), a struct or a class or interface with no [MessagePackObject] or + // [Union] and no collection shape (primitives, enums, float2, DateTime, ImmutableArray, ArraySegment, Memory, + // KeyValuePair, tuples, Lazy, unkeyed classes), and object, which is untyped on disk. + // Map a class or interface implementing IDictionary<,> or IReadOnlyDictionary<,>. + // Sequence an array or an IEnumerable class or interface. + // Owner a class, interface or struct MessagePack serializes by integer key. A keyed struct is walked as an owner; + // it is never an element (IsElement), since a boxed copy cannot carry a minted id. + private static (Kind Kind, Type Inner) Classify(Type declared) => Classes.GetOrAdd(declared, static type => + { + type = Nullable.GetUnderlyingType(type) ?? type; + if (type == typeof(string)) + return (Kind.Leaf, type); + if (IsWrittenWhole(type)) + return (Kind.Leaf, type); + if (IsKeyed(type)) + return (Kind.Owner, type); + if (type.IsValueType) + return (Kind.Leaf, type); + if (MapTypes(type) is { } map) + return (Kind.Map, map.Value); + if (ElementTypeOf(type) is { } element) + return (Kind.Sequence, element); + return (Kind.Leaf, type); + }); + + // A type MessagePack writes as one value the walk cannot look into: its own formatter ([MessagePackFormatter] on the type, + // inherited), [DataContract] on a type that is not [MessagePackObject] (the object resolver writes its members), or string + // keys (KeyAsPropertyName, or a [Key] with a string key on a member MessagePack writes: a private or [IgnoreMember] one is not written). Such a type is Leaf, even when it is also enumerable. + private static bool IsWrittenWhole(Type type) => + type.IsDefined(typeof(MessagePackFormatterAttribute), true) || + (type.IsDefined(typeof(DataContractAttribute), true) && !type.IsDefined(typeof(MessagePackObjectAttribute), true)) || + type.GetCustomAttribute(true)?.KeyAsPropertyName == true || + MembersOf(type).Any(member => member.GetCustomAttribute(true)?.StringKey != null); + + // A type MessagePack serializes by key: [MessagePackObject] or [Union]. + private static bool IsKeyed(Type type) => + type.IsDefined(typeof(MessagePackObjectAttribute), true) || type.IsDefined(typeof(UnionAttribute), true); + + // What an object list holds: a keyed class or interface. Never a struct. + private static bool IsElement(Type type) => !type.IsValueType && Classify(type).Kind == Kind.Owner; + + private static Type? ElementTypeOf(Type type) + { + if (type.IsArray) + return type.GetElementType(); + foreach (var candidate in type.GetInterfaces().Append(type)) + { + if (candidate.IsGenericType && candidate.GetGenericTypeDefinition() == typeof(IEnumerable<>)) + return candidate.GetGenericArguments()[0]; + } + + return null; + } + + // The key and value types of a map MessagePack writes as a map: IDictionary<,> or IReadOnlyDictionary<,>. + private static (Type Key, Type Value)? MapTypes(Type type) + { + foreach (var candidate in type.GetInterfaces().Append(type)) + { + if (candidate.IsGenericType && + (candidate.GetGenericTypeDefinition() == typeof(IDictionary<,>) || candidate.GetGenericTypeDefinition() == typeof(IReadOnlyDictionary<,>))) + { + var arguments = candidate.GetGenericArguments(); + return (arguments[0], arguments[1]); + } + } + + return null; + } + + private static readonly ConcurrentDictionary>> MapReaders = new(); + + // A map's entries read through the generic interface Classify found, so no map shape is recognised twice. + private static Func> MapEntries(Type map) => MapReaders.GetOrAdd(map, static type => + { + var (key, value) = MapTypes(type)!.Value; + return (Func>)Delegate.CreateDelegate( + typeof(Func>), + typeof(CultElementIds).GetMethod(nameof(ReadMap), BindingFlags.NonPublic | BindingFlags.Static)!.MakeGenericMethod(key, value)); + }); + + private static IEnumerable<(object? Key, object? Value)> ReadMap(object map) => + ((IEnumerable>)map).Select(entry => ((object?)entry.Key, (object?)entry.Value)); + } +} diff --git a/src/GameCult.Caching/CultSchemaConflictException.cs b/src/GameCult.Caching/CultSchemaConflictException.cs new file mode 100644 index 00000000..bbaff988 --- /dev/null +++ b/src/GameCult.Caching/CultSchemaConflictException.cs @@ -0,0 +1,34 @@ +using System; + +namespace GameCult.Caching +{ + /// + /// A write the catalog cannot describe: two schemas that share a schema id and disagree on the schema name (both arrived with the + /// records, neither registered), a record whose schema id no chosen entry publishes, or a write that would replace or remove a + /// . Nothing is written, so the store stays as it was. It names the id, the schema names involved + /// and a record key. + /// + public sealed class CultSchemaConflictException : InvalidOperationException + { + public CultSchemaConflictException(string message, string schemaId, string[] schemaNames, string recordKey) + : base(message) + { + SchemaId = schemaId; + SchemaNames = schemaNames; + RecordKey = recordKey; + } + + /// The same refusal when the reload it owes (what the store holds now, loaded into the cache) itself failed: that failure is the inner exception. + public CultSchemaConflictException(string message, string schemaId, string[] schemaNames, string recordKey, Exception? innerException) + : base(message, innerException) + { + SchemaId = schemaId; + SchemaNames = schemaNames; + RecordKey = recordKey; + } + + public string SchemaId { get; } + public string[] SchemaNames { get; } + public string RecordKey { get; } + } +} diff --git a/src/GameCult.Caching/CultStoreUnreadableException.cs b/src/GameCult.Caching/CultStoreUnreadableException.cs new file mode 100644 index 00000000..b3e43a4d --- /dev/null +++ b/src/GameCult.Caching/CultStoreUnreadableException.cs @@ -0,0 +1,22 @@ +using System; + +namespace GameCult.Caching +{ + /// + /// A store file this runtime cannot read: not exactly one complete store (truncated, bytes after it, a value that is not a + /// store, a missing or extra top-level slot), a format or record it does not know, or a body it cannot decode. Open, flush, + /// commit and CultMesh's single-file reads all refuse a file with this one exception, and a refused file is left as it was. + /// The underlying cause, where there is one, is the inner exception, and names the file. + /// + public sealed class CultStoreUnreadableException : NotSupportedException + { + public CultStoreUnreadableException(string message, string? path = null, Exception? innerException = null) + : base(message, innerException) + { + Path = path; + } + + /// The store file that was refused, or null when the bytes were read without a file. + public string? Path { get; } + } +} diff --git a/src/GameCult.Caching/CultVariants.cs b/src/GameCult.Caching/CultVariants.cs index ee5cc523..48909c19 100644 --- a/src/GameCult.Caching/CultVariants.cs +++ b/src/GameCult.Caching/CultVariants.cs @@ -28,6 +28,24 @@ namespace GameCult.Caching // (document payload, slot and encoded value pairs) public Func>, byte[]> Overlay { get; } + + /// + /// Decodes a document payload read from a store file, as a cache load decodes it: the ids a load mints are minted from the key + /// and their path, so every reader of one store mints the same ids, and a payload a cache write would refuse is refused here too. + /// It reads no file and holds no state. + /// + /// The registered document type. + /// The record key the payload was stored under. + /// The record's payload. + /// The payload holds an element id or list a cache load refuses. + public TDocument DecodeDocument(CultRecordKey key, byte[] payload) + where TDocument : class + { + if (payload == null) throw new ArgumentNullException(nameof(payload)); + var document = (TDocument)Deserialize(typeof(TDocument), typeof(TDocument), payload)!; + CultElementIds.Plan(typeof(TDocument), document, key.Value, deterministic: true).Apply(); + return document; + } } public enum CultOverrideOp diff --git a/src/GameCult.Caching/CultWriteConflictException.cs b/src/GameCult.Caching/CultWriteConflictException.cs new file mode 100644 index 00000000..61dd14e0 --- /dev/null +++ b/src/GameCult.Caching/CultWriteConflictException.cs @@ -0,0 +1,27 @@ +using System; +using System.Collections.Generic; + +namespace GameCult.Caching +{ + /// + /// A write refused because the store it would land in holds a variant and has changed since its writer last read it. A variant + /// resolves against other records, and a write never decodes a record it did not stage, so it cannot judge records another + /// writer changed: it writes only while every record header in the store equals what the writer last read or wrote. Nothing is + /// written. are the keys the write would have written or removed and the keys + /// whose durable header differs from what the writer last read, both ordinal-sorted. The cache has reloaded all of them as the + /// store holds them and forgotten the staged changes at those keys, so a retry is judged against that. When reloading failed, + /// nothing changed and is the cause. + /// + public sealed class CultWriteConflictException : InvalidOperationException + { + public CultWriteConflictException(string message, IReadOnlyList recordKeys, IReadOnlyList changedKeys, Exception? inner = null) + : base(message, inner) + { + RecordKeys = recordKeys; + ChangedKeys = changedKeys; + } + + public IReadOnlyList RecordKeys { get; } + public IReadOnlyList ChangedKeys { get; } + } +} diff --git a/src/GameCult.Caching/GameCult.Caching.csproj b/src/GameCult.Caching/GameCult.Caching.csproj index c3ec80ae..9819a079 100644 --- a/src/GameCult.Caching/GameCult.Caching.csproj +++ b/src/GameCult.Caching/GameCult.Caching.csproj @@ -5,10 +5,13 @@ - - - + + + + + + \ No newline at end of file diff --git a/src/GameCult.Caching/README.md b/src/GameCult.Caching/README.md index e9edb63d..fc5cd308 100644 --- a/src/GameCult.Caching/README.md +++ b/src/GameCult.Caching/README.md @@ -188,6 +188,40 @@ for (var row = 0; row < table.Count; row++) cache-owned physical table for hot loops and diagnostics; it is not a second domain model callers have to maintain. +## Element ids + +A document type that holds a list of objects gives every object type in that list +(union arms included) a string member marked `[CultElementId]`, next to its +`[Key]`. The id is what lets a variant override one list element instead of the +whole list. Lists of strings, numbers and other non-object values need none. + +The rule covers the shapes the id walk owns: arrays, `IEnumerable` classes and +interfaces, dictionary values, and integer-keyed `[MessagePackObject]` and `[Union]` +types. Any other holder is one whole value: `ImmutableArray`, `ArraySegment`, +`Memory`, `KeyValuePair`, tuples, `Lazy`, a class without `[MessagePackObject]`, +a `[DataContract]` or string-keyed type, and a type or member declaring +`[MessagePackFormatter]`. Its elements carry no element ids: a `[CultElementId]` +member inside it is a plain string, never minted or checked, and a variant can only +replace the holder whole. + +```csharp +[MessagePackObject] +public sealed class Line +{ + [Key(0)] [CultElementId] public string Id = string.Empty; + [Key(1)] public string Text = string.Empty; +} +``` + +Leave `Id` empty and the cache mints a random id when the record is written. An +id you set yourself must be exactly 12 lowercase hex characters +(`0123456789ab`); anything else is refused with a `CultElementIdException`. + +For readable ids, derive them: `[CultElementId(nameof(Name))]` makes an unset id +the text of that member (a string or number on the same type, unique within its +list), and a derived id may be any text. A readable name set by hand on a +non-derived list id is refused; the same member outside a list is unconstrained. + ## Persistence discipline Persistence stays manual by default. Mutations mark the cache and attached diff --git a/src/GameCult.Geometry/CultGeometryDocuments.cs b/src/GameCult.Geometry/CultGeometryDocuments.cs index 1ce73bdb..21383073 100644 --- a/src/GameCult.Geometry/CultGeometryDocuments.cs +++ b/src/GameCult.Geometry/CultGeometryDocuments.cs @@ -60,6 +60,10 @@ namespace GameCult.Geometry [MessagePackObject] public sealed class CultGeometryDomainNode { + [Key(7)] + [CultElementId] + public string Id { get; set; } = string.Empty; + [Key(0)] public string Name { get; set; } = string.Empty; @@ -105,6 +109,10 @@ namespace GameCult.Geometry [MessagePackObject] public sealed class CultGeometryFeatureClaim { + [Key(8)] + [CultElementId] + public string Id { get; set; } = string.Empty; + [Key(0)] public string Name { get; set; } = string.Empty; @@ -261,6 +269,10 @@ namespace GameCult.Geometry [MessagePackObject] public sealed class CultGeometryContributionRow { + [Key(13)] + [CultElementId] + public string Id { get; set; } = string.Empty; + [Key(0)] public string DomainKey { get; set; } = string.Empty; diff --git a/src/GameCult.Mesh/CultMesh.cs b/src/GameCult.Mesh/CultMesh.cs index 0d7ebd16..e3f23430 100644 --- a/src/GameCult.Mesh/CultMesh.cs +++ b/src/GameCult.Mesh/CultMesh.cs @@ -2732,11 +2732,9 @@ namespace GameCult.Mesh // runtime does not recognize by any alias) its payload's own embedded schema stamp, // alias-matched against TDocument's descriptor. The parse step // (CultNetDocumentRegistry.TryReadSchemaVersion) is the one decode rule shared with - // DecodeSnapshotDocuments; the match itself stays a direct alias-match against the caller's - // own descriptor rather than a global "which registered type owns this schema string" lookup, - // because that lookup is ambiguous when two document types alias the same schema id (as - // MeshNoteDocument/MeshNoteAliasDocument do in tests/GameCult.Mesh.Tests). No re-filter of the - // server's answer by schema alone, and no fallback that returns a record under the wrong key. + // DecodeSnapshotDocuments; the match itself is a direct alias-match against the caller's own + // descriptor. No re-filter of the server's answer by schema alone, and no fallback that + // returns a record under the wrong key. private static TDocument ReadDocumentFromSnapshotResponse( CultNetSnapshotResponseRawMessage response, string schemaId, diff --git a/src/GameCult.Mesh/CultMeshBodies.cs b/src/GameCult.Mesh/CultMeshBodies.cs index 0c740c18..a5b37387 100644 --- a/src/GameCult.Mesh/CultMeshBodies.cs +++ b/src/GameCult.Mesh/CultMeshBodies.cs @@ -1,3 +1,4 @@ +using GameCult.Caching; using System; using System.IO; using System.IO.MemoryMappedFiles; @@ -34,6 +35,10 @@ namespace GameCult.Mesh [MessagePackObject] public sealed class CultMeshBodyDescriptor { + [Key(13)] + [CultElementId] + public string Id { get; set; } = string.Empty; + [Key(0)] public string BodyId { get; set; } = string.Empty; [Key(1)] public string SchemaId { get; set; } = string.Empty; [Key(2)] public int LayoutVersion { get; set; } diff --git a/src/GameCult.Mesh/CultMeshCdn.cs b/src/GameCult.Mesh/CultMeshCdn.cs index 6ea4696f..7f47f291 100644 --- a/src/GameCult.Mesh/CultMeshCdn.cs +++ b/src/GameCult.Mesh/CultMeshCdn.cs @@ -87,6 +87,11 @@ namespace GameCult.Mesh /// CultCache record key for the referenced chunk document. [Key(3)] public string RecordKey { get; set; } = string.Empty; + + /// Element id: the decimal text of , which is unique within a manifest and never random. + [Key(4)] + [CultElementId(nameof(Offset))] + public string Id { get; set; } = string.Empty; } /// @@ -266,7 +271,8 @@ namespace GameCult.Mesh ChunkHash = chunkHash, Offset = offset, SizeBytes = count, - RecordKey = CultMeshCdnArtifactChunk.CreateRecordKey(chunk).Value + RecordKey = CultMeshCdnArtifactChunk.CreateRecordKey(chunk).Value, + Id = offset.ToString(CultureInfo.InvariantCulture) }); } @@ -285,7 +291,8 @@ namespace GameCult.Mesh ChunkHash = chunkHash, Offset = 0, SizeBytes = 0, - RecordKey = CultMeshCdnArtifactChunk.CreateRecordKey(chunk).Value + RecordKey = CultMeshCdnArtifactChunk.CreateRecordKey(chunk).Value, + Id = "0" }); } diff --git a/src/GameCult.Mesh/CultMeshDiscoveryService.cs b/src/GameCult.Mesh/CultMeshDiscoveryService.cs index 3f108e03..f75a1181 100644 --- a/src/GameCult.Mesh/CultMeshDiscoveryService.cs +++ b/src/GameCult.Mesh/CultMeshDiscoveryService.cs @@ -417,6 +417,10 @@ namespace GameCult.Mesh [MessagePackObject] public sealed class CultMeshDiscoveryCandidateDocument { + [Key(5)] + [CultElementId] + public string Id { get; set; } = string.Empty; + [Key(0)] public CultMeshVerseDescriptorMessage Descriptor { get; set; } = new CultMeshVerseDescriptorMessage(); [Key(1)] public string SourceId { get; set; } = ""; [Key(2)] public string ObservedAtUtc { get; set; } = ""; diff --git a/src/GameCult.Mesh/CultMeshEntityPrefabs.cs b/src/GameCult.Mesh/CultMeshEntityPrefabs.cs index 194c9360..afb0b0bf 100644 --- a/src/GameCult.Mesh/CultMeshEntityPrefabs.cs +++ b/src/GameCult.Mesh/CultMeshEntityPrefabs.cs @@ -80,6 +80,10 @@ namespace GameCult.Mesh [MessagePackObject] public sealed class CultMeshEntityPrefabComponent { + [Key(2)] + [CultElementId] + public string Id { get; set; } = string.Empty; + /// Portable component kind or runtime adapter type. [Key(0)] public string Type { get; set; } = string.Empty; @@ -95,6 +99,10 @@ namespace GameCult.Mesh [MessagePackObject] public sealed class CultMeshEntityPrefabAssetRef { + [Key(6)] + [CultElementId] + public string Id { get; set; } = string.Empty; + /// Stable source asset id inside the prefab package. [Key(0)] public string AssetId { get; set; } = string.Empty; @@ -126,6 +134,10 @@ namespace GameCult.Mesh [MessagePackObject] public sealed class CultMeshEntityPrefabNode { + [Key(10)] + [CultElementId] + public string Id { get; set; } = string.Empty; + /// Stable node id inside the package. [Key(0)] public string NodeId { get; set; } = string.Empty; diff --git a/src/GameCult.Mesh/CultMeshPrimitives.cs b/src/GameCult.Mesh/CultMeshPrimitives.cs index 58723950..500f9ef3 100644 --- a/src/GameCult.Mesh/CultMeshPrimitives.cs +++ b/src/GameCult.Mesh/CultMeshPrimitives.cs @@ -910,13 +910,10 @@ namespace GameCult.Mesh /// Gets whether this handle can submit a client prediction for the underlying document value. bool CanSubmitPrediction { get; } - - /// Creates a same-schema alias presentation for another CLR document type. - CultMeshDocumentHandle AsSchemaAlias() where TAlias : class; } /// - /// Typed reactive document handle with schema-aware alias conversion. + /// Typed reactive document handle. /// public sealed class CultMeshDocumentHandle : ICultMeshDocumentHandle where TDocument : class @@ -1117,39 +1114,6 @@ namespace GameCult.Mesh return ObserveAsync().ConfigureAwait(false).GetAwaiter().GetResult(); } - /// Creates a same-schema alias presentation for another CLR document type. - public CultMeshDocumentHandle AsSchemaAlias() where TAlias : class - { - var aliasDescriptor = CultDocumentRegistry.Shared.GetRequired(); - if (!string.Equals(Descriptor.SchemaName, aliasDescriptor.SchemaName, StringComparison.Ordinal) || - !string.Equals(Descriptor.SchemaVersion, aliasDescriptor.SchemaVersion, StringComparison.Ordinal)) - { - throw new InvalidOperationException( - $"Document type {typeof(TAlias).FullName} uses schema '{aliasDescriptor.SchemaName}' " + - $"version '{aliasDescriptor.SchemaVersion}', but handle '{DocumentId}' exposes " + - $"schema '{SchemaName}' version '{SchemaVersion}'."); - } - - var aliasFeed = new CultMeshLiveFeed( - DocumentId, - async (parameters, context) => ConvertDocument( - await Feed.SnapshotAsync(parameters, context).ConfigureAwait(false)), - (parameters, context) => Feed - .Watch(parameters, context) - .Select(ConvertDocument), - Sources, - RouteHint); - - Func? replace = _replace == null - ? null - : value => _replace(ConvertDocument(value)); - Func? submitPrediction = _submitPrediction == null - ? null - : value => _submitPrediction(ConvertDocument(value)); - - return new CultMeshDocumentHandle(aliasFeed.Bind(Context), replace, submitPrediction); - } - internal static TDocumentValue CloneDocument(TDocumentValue document) where TDocumentValue : class { @@ -1216,19 +1180,6 @@ namespace GameCult.Mesh "field"); } - internal static TTarget ConvertDocument(TSource document) - where TSource : class - where TTarget : class - { - if (document == null) throw new ArgumentNullException(nameof(document)); - if (document is TTarget alreadyTyped) - { - return alreadyTyped; - } - - var payload = CultDocumentMessagePackSerialization.SerializeUntyped(document, typeof(TSource)); - return (TTarget)CultDocumentMessagePackSerialization.DeserializeUntyped(typeof(TTarget), payload); - } } /// @@ -1912,11 +1863,12 @@ namespace GameCult.Mesh $"CultMesh document catalog does not expose a document for schema '{schema}'."); } - /// Looks up a typed document handle by CLR type or same-schema alias. + /// Looks up a typed document handle by CLR type. public bool TryGetDocument( out CultMeshDocumentHandle document) where TDocument : class { + // One type owns a schema, so the handle for TDocument's schema is the handle for TDocument. if (_documentsByType.TryGetValue(typeof(TDocument), out var untypedDocument) && untypedDocument is CultMeshDocumentHandle typedDocument) { @@ -1924,25 +1876,11 @@ namespace GameCult.Mesh return true; } - var descriptor = CultDocumentRegistry.Shared.GetRequired(); - if (_documentsBySchema.TryGetValue(descriptor.SchemaVersion, out var schemaDocument) || - _documentsBySchema.TryGetValue(descriptor.SchemaName, out schemaDocument!)) - { - if (schemaDocument is CultMeshDocumentHandle schemaTypedDocument) - { - document = schemaTypedDocument; - return true; - } - - document = schemaDocument.AsSchemaAlias(); - return true; - } - document = null!; return false; } - /// Looks up a typed document handle by CLR type or same-schema alias. + /// Looks up a typed document handle by CLR type. public CultMeshDocumentHandle Document() where TDocument : class { @@ -1953,70 +1891,70 @@ namespace GameCult.Mesh $"CultMesh document catalog does not expose a document for {typeof(TDocument).FullName}."); } - /// Reads one typed document snapshot by CLR type or same-schema alias. + /// Reads one typed document snapshot by CLR type. public Task LatestAsync() where TDocument : class { return Document().LatestAsync(); } - /// Reads one typed document synchronously by CLR type or same-schema alias. + /// Reads one typed document synchronously by CLR type. public TDocument Latest() where TDocument : class { return Document().Latest(); } - /// Gets whether one typed document can be replaced by CLR type or same-schema alias. + /// Gets whether one typed document can be replaced by CLR type. public bool CanReplace() where TDocument : class { return TryGetDocument(out var document) && document.CanReplace; } - /// Replaces one typed document by CLR type or same-schema alias. + /// Replaces one typed document by CLR type. public Task ReplaceAsync(TDocument value) where TDocument : class { return Document().ReplaceAsync(value); } - /// Gets whether one typed document can submit client predictions by CLR type or same-schema alias. + /// Gets whether one typed document can submit client predictions by CLR type. public bool CanSubmitPrediction() where TDocument : class { return TryGetDocument(out var document) && document.CanSubmitPrediction; } - /// Submits a client prediction for one typed document by CLR type or same-schema alias. + /// Submits a client prediction for one typed document by CLR type. public Task SubmitPredictionAsync(TDocument value) where TDocument : class { return Document().SubmitPredictionAsync(value); } - /// Gets an explicit authoritative writer by CLR type or same-schema alias. + /// Gets an explicit authoritative writer by CLR type. public CultMeshDocumentWriter AuthoritativeWriter() where TDocument : class { return Document().AuthoritativeWriter(); } - /// Gets an explicit prediction writer by CLR type or same-schema alias. + /// Gets an explicit prediction writer by CLR type. public CultMeshDocumentWriter PredictionWriter() where TDocument : class { return Document().PredictionWriter(); } - /// Watches one typed document by CLR type or same-schema alias. + /// Watches one typed document by CLR type. public Observable Watch() where TDocument : class { return Document().Watch(); } - /// Subscribes to one typed document by CLR type or same-schema alias. + /// Subscribes to one typed document by CLR type. public IDisposable Watch(Action onNext) where TDocument : class { @@ -2050,9 +1988,6 @@ namespace GameCult.Mesh /// Gets typed state sources this collection handle depends on, when known. IReadOnlyList Sources { get; } - - /// Creates a same-schema alias presentation for another CLR document type. - CultMeshCollectionHandle AsSchemaAlias() where TAlias : class; } /// @@ -2118,11 +2053,12 @@ namespace GameCult.Mesh $"CultMesh collection catalog does not expose a collection for schema '{schema}'."); } - /// Looks up a typed collection handle by CLR type or same-schema alias. + /// Looks up a typed collection handle by CLR type. public bool TryGetCollection( out CultMeshCollectionHandle collection) where TDocument : class { + // One type owns a schema, so the handle for TDocument's schema is the handle for TDocument. if (_collectionsByType.TryGetValue(typeof(TDocument), out var untypedCollection) && untypedCollection is CultMeshCollectionHandle typedCollection) { @@ -2130,26 +2066,11 @@ namespace GameCult.Mesh return true; } - var descriptor = CultDocumentRegistry.Shared.GetRequired(); - if (_collectionsBySchema.TryGetValue(descriptor.SchemaId, out var schemaCollection) || - _collectionsBySchema.TryGetValue(descriptor.SchemaVersion, out schemaCollection!) || - _collectionsBySchema.TryGetValue(descriptor.SchemaName, out schemaCollection!)) - { - if (schemaCollection is CultMeshCollectionHandle schemaTypedCollection) - { - collection = schemaTypedCollection; - return true; - } - - collection = schemaCollection.AsSchemaAlias(); - return true; - } - collection = null!; return false; } - /// Looks up a typed collection handle by CLR type or same-schema alias. + /// Looks up a typed collection handle by CLR type. public CultMeshCollectionHandle Collection() where TDocument : class { @@ -2160,21 +2081,21 @@ namespace GameCult.Mesh $"CultMesh collection catalog does not expose a collection for {typeof(TDocument).FullName}."); } - /// Reads one typed collection snapshot by CLR type or same-schema alias. + /// Reads one typed collection snapshot by CLR type. public Task> LatestAsync() where TDocument : class { return Collection().LatestAsync(); } - /// Watches one typed collection by CLR type or same-schema alias. + /// Watches one typed collection by CLR type. public Observable> WatchChanges() where TDocument : class { return Collection().WatchChanges(); } - /// Subscribes to one typed collection by CLR type or same-schema alias. + /// Subscribes to one typed collection by CLR type. public IDisposable WatchChanges(Action> onNext) where TDocument : class { @@ -2246,7 +2167,7 @@ namespace GameCult.Mesh } /// - /// Typed reactive collection handle with schema-aware alias conversion. + /// Typed reactive collection handle. /// public sealed class CultMeshCollectionHandle : ICultMeshCollectionHandle where TDocument : class @@ -2313,49 +2234,6 @@ namespace GameCult.Mesh if (onNext == null) throw new ArgumentNullException(nameof(onNext)); return WatchChanges().Subscribe(onNext); } - - /// Creates a same-schema alias presentation for another CLR document type. - public CultMeshCollectionHandle AsSchemaAlias() where TAlias : class - { - var aliasDescriptor = CultDocumentRegistry.Shared.GetRequired(); - if (!string.Equals(Descriptor.SchemaName, aliasDescriptor.SchemaName, StringComparison.Ordinal) || - !string.Equals(Descriptor.SchemaVersion, aliasDescriptor.SchemaVersion, StringComparison.Ordinal)) - { - throw new InvalidOperationException( - $"Document type {typeof(TAlias).FullName} uses schema '{aliasDescriptor.SchemaName}' " + - $"version '{aliasDescriptor.SchemaVersion}', but collection '{CollectionId}' exposes " + - $"schema '{SchemaName}' version '{SchemaVersion}'."); - } - - return new CultMeshCollectionHandle( - CollectionId, - async () => (await LatestAsync().ConfigureAwait(false)) - .Select(ConvertDocument) - .ToArray(), - () => WatchChanges().Select(change => new CultMeshCollectionChange( - change.Kind, - change.Key, - change.SchemaId, - change.Document == null ? null : ConvertDocument(change.Document), - change.PreviousDocument == null ? null : ConvertDocument(change.PreviousDocument), - change.Message)), - Sources, - RouteHint); - } - - private static TTarget ConvertDocument(TSource document) - where TSource : class - where TTarget : class - { - if (document == null) throw new ArgumentNullException(nameof(document)); - if (document is TTarget alreadyTyped) - { - return alreadyTyped; - } - - var payload = CultDocumentMessagePackSerialization.SerializeUntyped(document, typeof(TSource)); - return (TTarget)CultDocumentMessagePackSerialization.DeserializeUntyped(typeof(TTarget), payload); - } } /// diff --git a/src/GameCult.Mesh/CultMeshSingleFileDocuments.cs b/src/GameCult.Mesh/CultMeshSingleFileDocuments.cs index 17e0343d..37de5226 100644 --- a/src/GameCult.Mesh/CultMeshSingleFileDocuments.cs +++ b/src/GameCult.Mesh/CultMeshSingleFileDocuments.cs @@ -1,11 +1,9 @@ using System; -using System.Buffers; using System.IO; using System.Linq; using System.Security.Cryptography; using GameCult.Caching; using GameCult.Caching.MessagePack; -using MessagePack; namespace GameCult.Mesh { @@ -96,14 +94,9 @@ namespace GameCult.Mesh { if (document == null) throw new ArgumentNullException(nameof(document)); - var resolvedRegistry = registry ?? CultDocumentRegistry.Shared; - var descriptor = resolvedRegistry.GetRequired(); - WriteSingleFileDocumentPayload( - path, - key, - descriptor.ToCatalogEntry(), - storedAt, - CultDocumentMessagePackSerialization.SerializeUntyped(document, typeof(TDocument))); + if (string.IsNullOrWhiteSpace(path)) throw new ArgumentException("Value must be non-empty.", nameof(path)); + // The store decides element ids, the header and the copy of every other record, as a cache write does. + StoreAt(path).WriteDocument(key, document, storedAt, registry); } /// @@ -118,7 +111,8 @@ namespace GameCult.Mesh var resolvedRegistry = registry ?? CultDocumentRegistry.Shared; var descriptor = resolvedRegistry.GetRequired(); var payload = ReadSingleFileDocumentPayload(path, key, descriptor.SchemaId); - return (TDocument)CultDocumentMessagePackSerialization.DeserializeUntyped(typeof(TDocument), payload); + // A load refuses what a write refuses, and mints the ids a cache load of the same file would mint. + return CultCacheMessagePack.CreateCodec(resolvedRegistry).DecodeDocument(key, payload); } /// @@ -132,7 +126,7 @@ namespace GameCult.Mesh byte[] payload) { if (schema == null) throw new ArgumentNullException(nameof(schema)); - WriteSingleFileDocumentPayload(path, key, schema.ToCatalogEntry(payload), storedAt, payload); + WriteSingleFileDocumentPayload(path, key, schema.ToCatalogEntry(payload), storedAt, payload, holdsIds: null, registry: null); } /// @@ -146,7 +140,7 @@ namespace GameCult.Mesh if (string.IsNullOrWhiteSpace(path)) throw new ArgumentException("Value must be non-empty.", nameof(path)); if (string.IsNullOrWhiteSpace(expectedSchemaId)) throw new ArgumentException("Value must be non-empty.", nameof(expectedSchemaId)); - var snapshot = ReadSingleFileSnapshot(path); + var snapshot = StoreAt(path).ReadStoreSnapshot() ?? throw new FileNotFoundException($"CultCache document '{path}' does not exist.", path); var record = snapshot.Records.SingleOrDefault(candidate => string.Equals(candidate.Key, key.Value, StringComparison.Ordinal)) ?? throw new InvalidDataException($"CultCache document '{path}' does not contain record '{key.Value}'."); @@ -177,7 +171,7 @@ namespace GameCult.Mesh if (string.IsNullOrWhiteSpace(path)) throw new ArgumentException("Value must be non-empty.", nameof(path)); if (string.IsNullOrWhiteSpace(expectedSchemaId)) throw new ArgumentException("Value must be non-empty.", nameof(expectedSchemaId)); - var snapshot = ReadSingleFileSnapshot(path); + var snapshot = StoreAt(path).ReadStoreSnapshot() ?? throw new FileNotFoundException($"CultCache document '{path}' does not exist.", path); if (snapshot.Records.Length != 1) { throw new InvalidDataException( @@ -214,7 +208,9 @@ namespace GameCult.Mesh CultRecordKey key, CultSchemaCatalogEntry catalogEntry, string? storedAt, - byte[] payload) + byte[] payload, + bool? holdsIds, + CultDocumentRegistry? registry) { if (string.IsNullOrWhiteSpace(path)) throw new ArgumentException("Value must be non-empty.", nameof(path)); if (catalogEntry == null) throw new ArgumentNullException(nameof(catalogEntry)); @@ -222,140 +218,25 @@ namespace GameCult.Mesh throw new ArgumentException("Catalog entry must include a schema id.", nameof(catalogEntry)); payload ??= Array.Empty(); - var snapshot = new CultPersistedStoreSnapshot - { - SchemaCatalog = new[] { catalogEntry }, - Records = new[] + // The write replaces this one key and copies every other record and catalog entry forward as the store holds them, under + // the store's lock. A typed write sees its document, so its ids decide the header; a raw payload is opaque (holdsIds null), + // so a file already marked stays marked. A typed write attaches the registry its payload was encoded with, so the write + // decides for copied records that registry can read; a raw payload attaches none and decides nothing. A store this runtime + // cannot read refuses the write instead of being overwritten. + StoreAt(path).WriteRecord( + new CultPersistedRecord { - new CultPersistedRecord - { - Key = key.Value, - SchemaId = catalogEntry.SchemaId, - StoredAt = string.IsNullOrWhiteSpace(storedAt) ? DateTimeOffset.UtcNow.ToString("O") : storedAt!, - Payload = payload - } - } - }; - - var directory = Path.GetDirectoryName(path); - if (!string.IsNullOrWhiteSpace(directory)) - Directory.CreateDirectory(directory); - - // The write replaces the file, so it first reads what it would replace, under the lock a CultCache single-file - // store takes (.lock): a store this runtime cannot read (a newer format, or damage) or one holding any - // record but this key is refused, never overwritten. - using (AcquireStoreLock(path)) - { - RefuseToReplaceAStoreHoldingMore(path, key); - WriteFileAtomically(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); - } + Key = key.Value, + SchemaId = catalogEntry.SchemaId, + StoredAt = string.IsNullOrWhiteSpace(storedAt) ? DateTimeOffset.UtcNow.ToString("O") : storedAt!, + Payload = payload + }, + catalogEntry, + holdsIds, + registry); } - private static void RefuseToReplaceAStoreHoldingMore(string path, CultRecordKey key) - { - if (!File.Exists(path) || new FileInfo(path).Length == 0) - return; - - var existing = ReadSingleFileSnapshot(path); - var other = existing.Records.FirstOrDefault(record => - record.Variant != null || !string.Equals(record.Key, key.Value, StringComparison.Ordinal)); - if (other != null) - throw new InvalidOperationException( - $"CultCache document '{path}' holds record '{other.Key}' besides '{key.Value}'; a CultMesh single-file write " + - "replaces the whole file and would drop it. Write through a CultCache store instead."); - } - - private static FileStream AcquireStoreLock(string path) - { - var lockPath = path + ".lock"; - var started = System.Diagnostics.Stopwatch.StartNew(); - while (true) - { - try - { - return new FileStream(lockPath, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None, bufferSize: 1); - } - catch (IOException) when (started.Elapsed < TimeSpan.FromSeconds(30)) - { - System.Threading.Thread.Sleep(10); - } - } - } - - private static CultPersistedStoreSnapshot ReadSingleFileSnapshot(string path) - { - var bytes = File.ReadAllBytes(path); - CultPersistedStoreSnapshot snapshot; - try - { - snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(bytes, CultDocumentMessagePackSerialization.RequireReadableSingleFileStamp); - } - catch (Exception ex) when (ex is MessagePackSerializationException or InvalidOperationException) - { - snapshot = ReadLegacySingleFileSnapshot(bytes); - } - - CultDocumentMessagePackSerialization.RequireSingleFileFormat(snapshot); - return snapshot; - } - - private static CultPersistedStoreSnapshot ReadLegacySingleFileSnapshot(byte[] bytes) - { - var reader = new MessagePackReader(bytes); - var fieldCount = reader.ReadArrayHeader(); - var snapshot = new CultPersistedStoreSnapshot(); - if (fieldCount > 0) - snapshot.FormatVersion = reader.ReadString() - ?? throw new NotSupportedException("Store snapshot declares no format version."); - - if (fieldCount > 1) - { - var catalogCount = reader.ReadArrayHeader(); - snapshot.SchemaCatalog = new CultSchemaCatalogEntry[catalogCount]; - for (var index = 0; index < catalogCount; index++) - snapshot.SchemaCatalog[index] = ReadLegacySchemaCatalogEntry(ref reader); - } - - if (fieldCount > 2) - { - var recordCount = reader.ReadArrayHeader(); - snapshot.Records = new CultPersistedRecord[recordCount]; - for (var index = 0; index < recordCount; index++) - snapshot.Records[index] = CultDocumentMessagePackSerialization.ReadPersistedRecord(ref reader); - } - - for (var index = 3; index < fieldCount; index++) - reader.Skip(); - - return snapshot; - } - - private static CultSchemaCatalogEntry ReadLegacySchemaCatalogEntry(ref MessagePackReader reader) - { - var fieldCount = reader.ReadArrayHeader(); - var entry = new CultSchemaCatalogEntry(); - if (fieldCount > 0) entry.SchemaId = reader.ReadString() ?? string.Empty; - if (fieldCount > 1) entry.SchemaName = reader.ReadString() ?? string.Empty; - if (fieldCount > 2) entry.SchemaVersion = reader.ReadString() ?? string.Empty; - if (fieldCount > 3) entry.CanonicalSchemaJson = reader.ReadString() ?? string.Empty; - if (fieldCount > 4) reader.Skip(); - if (fieldCount > 5) entry.ContentHash = reader.ReadString() ?? string.Empty; - if (fieldCount > 6) - { - var memberCount = reader.ReadArrayHeader(); - for (var index = 0; index < memberCount; index++) - reader.Skip(); - } - - for (var index = 7; index < fieldCount; index++) - reader.Skip(); - - entry.CompatibleSchemaIds = string.IsNullOrWhiteSpace(entry.SchemaId) - ? Array.Empty() - : new[] { entry.SchemaId }; - entry.Members = Array.Empty(); - return entry; - } + private static SingleFileMessagePackBackingStore StoreAt(string path) => new SingleFileMessagePackBackingStore(path); private static bool PublishesSchema(CultSchemaCatalogEntry[] catalog, string schemaId) { @@ -363,14 +244,5 @@ namespace GameCult.Mesh string.Equals(entry.SchemaId, schemaId, StringComparison.Ordinal) || entry.CompatibleSchemaIds.Any(candidate => string.Equals(candidate, schemaId, StringComparison.Ordinal))); } - - private static void WriteFileAtomically(string path, byte[] bytes) - { - var tempPath = path + ".tmp"; - File.WriteAllBytes(tempPath, bytes); - if (File.Exists(path)) - File.Delete(path); - File.Move(tempPath, path); - } } } diff --git a/src/GameCult.Mesh/CultMeshSnapshots.cs b/src/GameCult.Mesh/CultMeshSnapshots.cs index 58aede9d..c76f82bc 100644 --- a/src/GameCult.Mesh/CultMeshSnapshots.cs +++ b/src/GameCult.Mesh/CultMeshSnapshots.cs @@ -979,11 +979,11 @@ namespace GameCult.Mesh var canDeserializeWithBinding = binding != null && typeof(TDocument).IsAssignableFrom(binding.DocumentType); - var canDeserializeAsSchemaAlias = + var canDeserializeByWireSchema = CultNetSchemaAliasMatching.Matches(record.SchemaId, descriptor) || (binding != null && CultNetSchemaAliasMatching.Matches(binding.SchemaId, descriptor)) || CultNetDocumentRegistry.PayloadMatchesSchema(record.Payload, descriptor); - if (!canDeserializeWithBinding && !canDeserializeAsSchemaAlias) + if (!canDeserializeWithBinding && !canDeserializeByWireSchema) continue; if (!string.Equals(record.PayloadEncoding, "messagepack", StringComparison.Ordinal)) diff --git a/src/GameCult.Mesh/docs/public-api.md b/src/GameCult.Mesh/docs/public-api.md index e9f19ab7..072e6c73 100644 --- a/src/GameCult.Mesh/docs/public-api.md +++ b/src/GameCult.Mesh/docs/public-api.md @@ -173,28 +173,22 @@ portable machinery that makes it feel native. the normal shard log. Callers can bind a projected live feed, a local `CultCache` record, a distributed `CultNetDatabase` record, or a `CultMeshNode` record through the same surface instead of walking transport - and projection layers themselves. Same-schema CLR aliases can be requested - directly from node/database/snapshot surfaces; CultMesh resolves the canonical - registered document by shared `[CultDocument(schemaName, schemaVersion)]` - identity and uses the shared CultCache MessagePack codec for - read/watch/replace/prediction conversion. `AsSchemaAlias()` remains - available when a caller already has a handle and wants to project it. + and projection layers themselves. One CLR type owns each + `[CultDocument(schemaName, schemaVersion)]` schema; the registry refuses a + second type for the same schema with `CultSchemaConflictException`. - `CultMeshDocumentCatalog` is the schema-aware lookup edge for a set of - document handles. It indexes handles by CLR document type, schema name, and - schema version, and resolves same-schema CLR aliases by delegating to - `CultMeshDocumentHandle.AsSchemaAlias()`. Domain facades - can expose `Document()`, `DocumentBySchema(...)`, `LatestAsync()`, and - `Watch()` without owning schema dictionaries or alias serializers. Mutable - facades can also call `ReplaceAsync(...)` or - `SubmitPredictionAsync(...)` directly on the catalog when they only need - the schema-aliased operation and not the intermediate handle. + document handles. It indexes handles by CLR document type, schema id, schema + name, and schema version. Domain facades can expose `Document()`, + `DocumentBySchema(...)`, `LatestAsync()`, and `Watch()` without owning + schema dictionaries. Mutable facades can also call `ReplaceAsync(...)` or + `SubmitPredictionAsync(...)` directly on the catalog when they do not need + the intermediate handle. - `CultMeshCollectionHandle` is the typed reactive edge for multi-record state. It exposes one `LatestAsync()` collection snapshot and a `WatchChanges()` stream while hiding whether the backing source is an in-process `CultCache`, a distributed `CultNetDatabase`, or a `CultMeshNode`. Collection handles can represent all records of a document type, a named - document view, or an indexed value view, and they support the same - same-schema CLR alias conversion as document handles. + document view, or an indexed value view. - `CultMeshCollectionCatalog` is the collection sibling of `CultMeshDocumentCatalog`. It indexes typed collection handles by CLR document type, schema id, schema name, and schema version, then exposes @@ -258,19 +252,17 @@ var objects = await visibleObjects.ExecuteAsync(viewportRequest); await movePilot.InvokeAsync(moveRequest, idempotencyKey); ``` -A single typed document can be surfaced the same way. The caller asks for the -typed view it wants; if that type is a same-schema alias of the registered -canonical document, CultMesh still opens the correct backing record: +A single typed document can be surfaced the same way: ```csharp -var stockForUi = CultMesh.Document( +var stockHandle = CultMesh.Document( node, new CultRecordKey("station:starbridge:stock"), verse); -var stock = await stockForUi.LatestAsync(); -using var watch = stockForUi.Watch(RenderStock); -await stockForUi.SetAsync(updatedStockFromUi); +var stock = await stockHandle.LatestAsync(); +using var watch = stockHandle.Watch(RenderStock); +await stockHandle.SetAsync(updatedStock); ``` Projected or derived documents can use the same handle shape with explicit @@ -291,8 +283,6 @@ var cockpit = CultMesh.Document( var current = await cockpit.LatestAsync(); using var subscription = cockpit.Watch(next => RenderCockpit(next)); - -var daemonAlias = cockpit.AsSchemaAlias(); ``` Remote CultNet snapshots use the same document handle surface. The caller names @@ -301,20 +291,18 @@ id, response filtering, raw MessagePack payload decode, route metadata, and polling watch fallback: ```csharp -var remoteHealth = CultMesh.DocumentFromPeerSnapshot( +var remoteHealth = CultMesh.DocumentFromPeerSnapshot( "cultnet://daemon.local:3075", "daemon:aetheria.health.v1", verse); -var uiHealth = await remoteHealth.LatestAsync(); +var health = await remoteHealth.LatestAsync(); ``` -Snapshot requests are record-key-first so same logical documents can still be -read when another runtime generated a different schema id for a compatible -schema alias. The decode path prefers exact schema id matches when present and -falls back to the requested record key. Ask for the runtime-facing alias type -directly; `AsSchemaAlias()` is for adapting handles that were already -constructed elsewhere. +Snapshot requests are record-key-first so the same logical document can still be +read when another runtime generated a different schema id for it. The decode +path prefers exact schema id matches when present and falls back to the +requested record key. A domain facade can still collect existing handles into one schema-aware catalog when it is composing pre-built surfaces: @@ -326,7 +314,7 @@ var documents = CultMesh.Documents( zoneContacts); var refit = await documents.LatestAsync(); -var uiDocking = documents.Document(); +var docking = documents.Document(); var bySchema = documents.DocumentBySchema("gamecult.aetheria.station_refit.v1"); ``` @@ -341,11 +329,9 @@ var allies = CultMesh.CollectionByIndex( var currentAllies = await allies.LatestAsync(); using var changes = allies.WatchChanges(change => UpdateRoster(change)); -var uiAllies = allies.AsSchemaAlias(); - var collections = CultMesh.Collections(allies); -var uiRoster = await collections.LatestAsync(); -using var uiRosterChanges = collections.WatchChanges(RenderRoster); +var roster = await collections.LatestAsync(); +using var rosterChanges = collections.WatchChanges(RenderRoster); ``` ```ts diff --git a/src/GameCult.Networking/CultNetDocumentRegistry.cs b/src/GameCult.Networking/CultNetDocumentRegistry.cs index 62691a3e..58b7ca36 100644 --- a/src/GameCult.Networking/CultNetDocumentRegistry.cs +++ b/src/GameCult.Networking/CultNetDocumentRegistry.cs @@ -165,10 +165,31 @@ namespace GameCult.Networking } /// - /// Registers a document binding. + /// Registers a document binding. One binding owns a document type and a schema id: a binding claiming a type or a schema + /// id another binding already holds is refused with , naming the new binding's + /// schema id and both document types, and nothing is bound. Binding a type to the schema id it already holds replaces + /// that binding. /// public CultNetDocumentRegistry Register(CultNetDocumentBinding binding) { + var holder = + _bindingsByType.TryGetValue(binding.DocumentType, out var sameType) && + !string.Equals(sameType.SchemaId, binding.SchemaId, StringComparison.Ordinal) + ? sameType + : _bindingsBySchemaId.TryGetValue(binding.SchemaId, out var sameSchema) && + sameSchema.DocumentType != binding.DocumentType + ? sameSchema + : null; + if (holder != null) + { + throw new CultSchemaConflictException( + $"CultNet schema id \"{holder.SchemaId}\" is already bound to type \"{holder.DocumentType.FullName}\"; " + + $"type \"{binding.DocumentType.FullName}\" (schema id \"{binding.SchemaId}\") cannot also be bound.", + binding.SchemaId, + new[] { holder.DocumentType.FullName ?? holder.DocumentType.Name, binding.DocumentType.FullName ?? binding.DocumentType.Name }, + string.Empty); + } + _bindingsBySchemaId[binding.SchemaId] = binding; _bindingsByType[binding.DocumentType] = binding; return this; diff --git a/src/GameCult.Networking/CultWitnessArtifactBundle.cs b/src/GameCult.Networking/CultWitnessArtifactBundle.cs index 045ebfbb..6baa48bb 100644 --- a/src/GameCult.Networking/CultWitnessArtifactBundle.cs +++ b/src/GameCult.Networking/CultWitnessArtifactBundle.cs @@ -104,6 +104,10 @@ namespace GameCult.Networking [MessagePackObject] public sealed class CultWitnessContractPin { + [Key(4)] + [CultElementId] + public string Id { get; set; } = string.Empty; + /// /// Gets or sets the contract role inside the bundle. /// @@ -135,6 +139,10 @@ namespace GameCult.Networking [MessagePackObject] public sealed class CultWitnessArtifactEntry { + [Key(6)] + [CultElementId] + public string Id { get; set; } = string.Empty; + /// /// Gets or sets the artifact role inside the witness bundle. /// @@ -178,6 +186,10 @@ namespace GameCult.Networking [MessagePackObject] public sealed class CultWitnessTimingEntry { + [Key(5)] + [CultElementId] + public string Id { get; set; } = string.Empty; + /// /// Gets or sets the pipeline stage name. /// diff --git a/tests/GameCult.Caching.ConsumerBuild/Program.cs b/tests/GameCult.Caching.ConsumerBuild/Program.cs index b0f15973..60376068 100644 --- a/tests/GameCult.Caching.ConsumerBuild/Program.cs +++ b/tests/GameCult.Caching.ConsumerBuild/Program.cs @@ -1,10 +1,69 @@ using System.Collections.Generic; +using System.Linq; using GameCult.Caching; using MessagePack; // RefKeyedDictionaryCompilesInAConsumer: this project building is the test. System.Console.WriteLine(new ConsumerFaction().Standing.Count); +// An outside store kind versions its foreign records: this project has no InternalsVisibleTo, so building and running it pins that +// Foreign(record, catalog, version) is all a store kind outside the library needs for ExpectUnchanged to hold over them. +using var outside = new CultCache(CultDocumentRegistry.ForTypes(new[] { typeof(ConsumerFaction) })); +outside.AddBackingStore(new OutsideStore(), typeof(ConsumerFaction)); +var committed = outside.Commit(batch => +{ + batch.ExpectUnchanged(); + batch.Upsert(new ConsumerFaction { Name = "a" }, new CultRecordHandle(new CultRecordKey("a"))); +}); +if (!committed) + return 1; + +// A store file is written and read without a cache through the public members: this project has no InternalsVisibleTo, so +// building it pins that ReadStoreSnapshot, WriteRecord and WriteDocument, and CultCodec.DecodeDocument, are public API. +var storePath = System.IO.Path.Combine(System.IO.Path.GetTempPath(), "consumer-build-" + System.Guid.NewGuid().ToString("N") + ".cc"); +try +{ + var entry = new CultSchemaCatalogEntry { SchemaId = "outside.written", SchemaName = "outside.written" }; + var store = new GameCult.Caching.MessagePack.SingleFileMessagePackBackingStore(storePath); + store.WriteRecord(new CultPersistedRecord { Key = "r", SchemaId = entry.SchemaId, StoredAt = "t", Payload = new byte[] { 0x90 } }, entry); + var snapshot = store.ReadStoreSnapshot(); + if (snapshot == null || snapshot.Records.Length != 1 || snapshot.Records[0].Key != "r") + return 1; + var typedRegistry = CultDocumentRegistry.ForTypes(new[] { typeof(ConsumerFaction) }); + store.WriteDocument(new CultRecordKey("f"), new ConsumerFaction { Name = "typed" }, registry: typedRegistry); + var typed = store.ReadStoreSnapshot()!.Records.Single(record => record.Key == "f"); + if (GameCult.Caching.MessagePack.CultCacheMessagePack.CreateCodec(typedRegistry).DecodeDocument(new CultRecordKey("f"), typed.Payload).Name != "typed") + return 1; +} +finally +{ + System.IO.File.Delete(storePath); +} +return 0; + +// Lists one record under an id no type claims, at version "v1", and commits only when the conditions hold over it. +public sealed class OutsideStore : CacheBackingStore +{ + private static readonly CultSchemaCatalogEntry Entry = new() { SchemaId = "outside.unclaimed", SchemaName = "outside.unclaimed" }; + + public override void PullAll() + { + var record = new CultPersistedRecord { Key = "w", SchemaId = Entry.SchemaId, StoredAt = "t" }; + SetForeignRecords(new[] { Foreign(record, new[] { Entry }, "v1")! }); + } + + public override void Push(CultStoredDocument entry) { } + + public override void Delete(CultStoredDocument entry) { } + + public override void PushAll() { } + + public override CultCommitOutcome CommitBatch(CultCommitRequest request, bool wait) => + request.ConditionsHold(new Dictionary { ["w"] = "v1" }, Entries.Values, ForeignRecords) + ? CultCommitOutcome.Committed + : CultCommitOutcome.Mismatch; +} + [CultDocument("tests.consumer_faction", "tests.consumer_faction.v1")] [MessagePackObject] public sealed class ConsumerFaction diff --git a/tests/GameCult.Caching.InteropPeer/Program.cs b/tests/GameCult.Caching.InteropPeer/Program.cs index 20e2c62c..3c8eb92b 100644 --- a/tests/GameCult.Caching.InteropPeer/Program.cs +++ b/tests/GameCult.Caching.InteropPeer/Program.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Globalization; +using System.IO; using System.Linq; using System.Text.Json; using System.Threading.Tasks; @@ -16,7 +17,7 @@ static async Task ProgramMainAsync(string[] args) { if (args.Length == 0) { - throw new InvalidOperationException("Expected mode: write | read | hold-lock | write-routed "); + throw new InvalidOperationException("Expected mode: write | read | rewrite | hold-lock | write-routed "); } var mode = args[0]; @@ -38,6 +39,18 @@ static async Task ProgramMainAsync(string[] args) case "read": await ReadAsync(file); return 0; + case "rewrite": + await RewriteAsync(file); + return 0; + case "write-deck": + await WriteDeckAsync(file); + return 0; + case "read-deck": + await ReadDeckAsync(file); + return 0; + case "edit-in-place": + await EditInPlaceAsync(file, RequireArg(options, "kind")); + return 0; case "hold-lock": HoldLock(file, int.Parse(RequireArg(options, "ms"), CultureInfo.InvariantCulture)); return 0; @@ -94,6 +107,113 @@ static async Task ReadAsync(string file) WriteJsonLine(note); } +// Puts the stored note back: the cache stamps it with the registered schema id, whatever id it was stored under. +static async Task RewriteAsync(string file) +{ + var cache = BuildCache(file); + await cache.PullAllBackingStoresAsync(); + var note = cache.AllEntries.OfType().Single(); + await cache.UpsertAsync(note, new CultRecordHandle(new CultRecordKey(note.DocumentId))); + cache.FlushAllBackingStores(); + WriteJsonLine(note); +} + +// A record with an object list: the cache mints each mark's element id, and every runtime reads it as an ordinary member. +static async Task WriteDeckAsync(string file) +{ + var cache = BuildCache(file); + await cache.PullAllBackingStoresAsync(); + var deck = new CultCacheInteropDeck + { + DocumentId = "deck:csharp", + Marks = [new CultCacheInteropMark { Label = "twin" }, new CultCacheInteropMark { Label = "twin" }, new CultCacheInteropMark { Label = "other" }] + }; + await cache.AddAsync(deck, new CultRecordHandle(new CultRecordKey(deck.DocumentId))); + cache.FlushAllBackingStores(); + WriteDeckJsonLine(deck); +} + +static async Task ReadDeckAsync(string file) +{ + var cache = BuildCache(file); + await cache.PullAllBackingStoresAsync(); + WriteDeckJsonLine(cache.AllEntries.OfType().Single()); +} + +static void WriteDeckJsonLine(CultCacheInteropDeck deck) => + Console.Out.WriteLine(JsonSerializer.Serialize(new { documentId = deck.DocumentId, ids = deck.Marks.Select(mark => mark.Id).ToArray() })); + +// Edits a stored record in place, as a consumer holding the object would, and writes it: the file must hold the edit, in whatever +// runtime reads it. payload and override go through the public snapshot API (a decoded record's bytes edited, then +// SerializeSnapshot); collection and object edit the document the cache holds and put the same instance back. +static async Task EditInPlaceAsync(string file, string kind) +{ + switch (kind) + { + case "payload": + { + var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(file)); + OverwriteInPlace(snapshot.Records.Single().Payload, "wrote", "WROTE"); + File.WriteAllBytes(file, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + break; + } + case "override": + { + var cache = CultCacheMessagePack.Create(file, new CultCacheOpenOptions { Registry = CultDocumentRegistry.ForTypes(new[] { typeof(CultCacheInteropNote) }) }); + await cache.PullAllBackingStoresAsync(); + var note = cache.AllEntries.OfType().Single(); + await cache.UpsertVariantAsync( + new CultRecordKey("note:variant"), + new CultRecordKey(note.DocumentId), + new[] + { + cache.Override(nameof(CultCacheInteropNote.DocumentId), "note:variant"), + cache.Override(nameof(CultCacheInteropNote.Title), "variant title") + }); + cache.FlushAllBackingStores(); + var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(file)); + var variant = snapshot.Records.Single(record => record.Variant != null).Variant!; + OverwriteInPlace(variant.Overrides.Single(entry => entry.Path[0].Slot == 3).Value, "variant", "VARIANT"); + File.WriteAllBytes(file, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + break; + } + case "collection": + { + var cache = BuildCache(file); + await cache.PullAllBackingStoresAsync(); + var note = cache.AllEntries.OfType().Single(); + note.Tags[0] = "EDITED"; + await cache.UpsertAsync(note, new CultRecordHandle(new CultRecordKey(note.DocumentId))); + cache.FlushAllBackingStores(); + break; + } + case "object": + { + var cache = BuildCache(file); + await cache.PullAllBackingStoresAsync(); + var deck = cache.AllEntries.OfType().Single(); + deck.Marks[0].Label = "EDITED"; + await cache.UpsertAsync(deck, new CultRecordHandle(new CultRecordKey(deck.DocumentId))); + cache.FlushAllBackingStores(); + break; + } + default: + throw new InvalidOperationException($"Unknown edit kind {kind}."); + } + + Console.Out.WriteLine(JsonSerializer.Serialize(new { kind })); +} + +static void OverwriteInPlace(byte[] bytes, string from, string to) +{ + var was = System.Text.Encoding.UTF8.GetBytes(from); + var now = System.Text.Encoding.UTF8.GetBytes(to); + var at = bytes.AsSpan().IndexOf(was); + if (at < 0 || now.Length != was.Length) + throw new InvalidOperationException($"Cannot overwrite '{from}' with '{to}' in place."); + now.CopyTo(bytes, at); +} + // Two routed single-file stores: each file is a complete single-store snapshot holding only its own type. static void WriteRouted(string catalogFile, string runFile) { @@ -172,7 +292,7 @@ static void WriteJsonLine(CultCacheInteropNote note) })); } -[CultDocument("cultcache.interop-note", "cultcache.interop_note.v1")] +[CultDocument("cultcache.interop-note", "cultcache.interop_note.v1", CompatibleSchemaIds = new[] { "cultcache.interop-note.legacy-id" })] [MessagePackObject] public sealed class CultCacheInteropNote { @@ -193,3 +313,18 @@ public sealed class CultCacheInteropRunNote [Key(2)] public string AuthorRuntimeId { get; set; } = string.Empty; [Key(3)] public string Body { get; set; } = string.Empty; } + +[CultDocument("cultcache.interop-deck", "cultcache.interop_deck.v1")] +[MessagePackObject] +public sealed class CultCacheInteropDeck +{ + [Key(0)] [CultName] public string DocumentId { get; set; } = string.Empty; + [Key(1)] public List Marks { get; set; } = new(); +} + +[MessagePackObject] +public sealed class CultCacheInteropMark +{ + [Key(0)] public string Label { get; set; } = string.Empty; + [Key(1)] [CultElementId] public string Id { get; set; } = string.Empty; +} diff --git a/tests/GameCult.Caching.Tests/AssemblyInfo.cs b/tests/GameCult.Caching.Tests/AssemblyInfo.cs index 0dd8b125..708f0a09 100644 --- a/tests/GameCult.Caching.Tests/AssemblyInfo.cs +++ b/tests/GameCult.Caching.Tests/AssemblyInfo.cs @@ -6,7 +6,10 @@ using MessagePack.Formatters; [assembly: CultCacheFormatterResolver(typeof(PairResolver))] [assembly: CultCacheFormatterResolver(typeof(UnionArmGuardTests.StubArmResolver))] +[assembly: CultCacheFormatterResolver(typeof(UnionArmGuardTests.ClaimArmResolver))] [assembly: CultCacheFormatterResolver(typeof(UnionArmGuardTests.OrderResolver))] +[assembly: CultCacheFormatterResolver(typeof(UnionArmGuardTests.ListClaimResolver))] +[assembly: CultCacheFormatterResolver(typeof(UnionArmGuardTests.FbLaterClaimResolver))] namespace GameCult.Caching.Tests { diff --git a/tests/GameCult.Caching.Tests/C2aGateWriter.cs b/tests/GameCult.Caching.Tests/C2aGateWriter.cs new file mode 100644 index 00000000..9a018a3e --- /dev/null +++ b/tests/GameCult.Caching.Tests/C2aGateWriter.cs @@ -0,0 +1,140 @@ +#nullable enable +using System; +using System.IO; +using System.Linq; +using GameCult.Caching; +using GameCult.Caching.MessagePack; +using NUnit.Framework; + +namespace GameCult.Caching.Tests +{ + // The writer half of the C2a format gate. It writes a record that holds element ids through CultCacheMessagePack.Create, once + // as a single-file store and once as a directory store, and asserts the headers C2a stamps. old-reader-refusal.sh gate runs it + // with CULTLIB_GATE_OUT set, so the 2.0.1 reader probes the very bytes this tree writes; the committed vectors are one such run. + public class C2aGateWriter + { + private const string SingleName = "c2a-written-ids.msgpack"; + private const string DirectoryName = "c2a-written-ids-directory"; + private const string ManifestName = "store.cc"; + private const string DirectoryHeader = "cultcache.store.v5.directory-content-addressed-pages"; + + private static readonly CultDocumentRegistry Registry = CultDocumentRegistry.ForTypes(new[] { typeof(ElementIdTests.IdDeck) }); + + // Every id is the author's, so the written bytes and the decoded ids do not depend on minting. + private static readonly string[] Ids = { "00000000c2a1", "00000000c2a2", "00000000c2a3", "00000000c2a4" }; + + private static ElementIdTests.IdDeck Deck() => new() + { + Name = "gate", + Reels = + { + new ElementIdTests.IdReel + { + Label = "reel", + Id = Ids[0], + Marks = { new ElementIdTests.IdMark { Text = "m", Id = Ids[1] }, new ElementIdTests.IdMark { Text = "m", Id = Ids[2] } } + } + }, + Moves = { new ElementIdTests.IdSlash { Reach = 1, Id = Ids[3] } } + }; + + private static string[] IdsOf(ElementIdTests.IdDeck deck) => + deck.Reels.Select(reel => reel.Id) + .Concat(deck.Reels.SelectMany(reel => reel.Marks.Select(mark => mark.Id))) + .Concat(deck.Moves.Select(move => move.Id)) + .ToArray(); + + // The one instant every record is stamped with, so the bytes this writes are the same on every run and the committed vectors + // can be compared to them byte for byte. Regenerate the vectors with CULTLIB_GATE_OUT (see C2aStamps...) if C2a's format moves. + private static readonly DateTimeOffset StoredAt = new(2026, 10, 27, 0, 0, 0, TimeSpan.Zero); + + private static void Write(string path, bool directory) + { + using var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry, UseDirectoryStore = directory }); + cache.Clock = () => StoredAt; + cache.Commit(batch => batch.Upsert(typeof(ElementIdTests.IdDeck), Deck(), new CultRecordKey("gate"))); + } + + private static string HeaderOf(string path) => + CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)).FormatVersion; + + private static string[] ReadIds(string path) + { + using var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry }); + return IdsOf(cache.Get(new CultRecordKey("gate"))!); + } + + private static string Vectors() + { + for (var dir = new DirectoryInfo(TestContext.CurrentContext.TestDirectory); dir != null; dir = dir.Parent) + { + var vectors = Path.Combine(dir.FullName, "tests", "vectors", "document-variants-c2a"); + if (Directory.Exists(vectors)) return vectors; + } + + throw new DirectoryNotFoundException($"tests/vectors/document-variants-c2a not found above {TestContext.CurrentContext.TestDirectory}."); + } + + // Every file under root by relative path and bytes; a store's lock files hold no state and are not compared. + private static (string Path, byte[] Bytes)[] Tree(string root) => + Directory.GetFiles(root, "*", SearchOption.AllDirectories) + .Where(file => !file.EndsWith(".lock", StringComparison.Ordinal)) + .Select(file => (Path.GetRelativePath(root, file).Replace(Path.DirectorySeparatorChar, '/'), File.ReadAllBytes(file))) + .OrderBy(entry => entry.Item1, StringComparer.Ordinal) + .ToArray(); + + private string _temp = string.Empty; + + [SetUp] + public void CreateTemp() + { + _temp = Path.Combine(Path.GetTempPath(), $"cultlib-c2a-gate-{Guid.NewGuid():N}"); + Directory.CreateDirectory(_temp); + } + + [TearDown] + public void DeleteTemp() + { + if (Directory.Exists(_temp)) + Directory.Delete(_temp, recursive: true); + } + + [Test] + public void C2aStampsV3OnASingleFileStoreAndV5OnADirectoryStoreThatHoldIds() + { + var output = Environment.GetEnvironmentVariable("CULTLIB_GATE_OUT"); + var root = string.IsNullOrEmpty(output) ? _temp : output; + Directory.CreateDirectory(root); + var single = Path.Combine(root, SingleName); + var directory = Path.Combine(root, DirectoryName); + File.Delete(single); + if (Directory.Exists(directory)) Directory.Delete(directory, recursive: true); + Directory.CreateDirectory(directory); + + Write(single, directory: false); + Write(Path.Combine(directory, ManifestName), directory: true); + + Assert.That(HeaderOf(single), Is.EqualTo(CultPersistedStoreSnapshot.FormatV3)); + Assert.That(HeaderOf(Path.Combine(directory, ManifestName)), Is.EqualTo(DirectoryHeader)); + Assert.That(ReadIds(single), Is.EqualTo(Ids)); + } + + [Test] + public void TheCommittedVectorsAreWhatC2aWrites() + { + var live = Path.Combine(_temp, "live"); + var liveDirectory = Path.Combine(live, DirectoryName); + Directory.CreateDirectory(liveDirectory); + Write(Path.Combine(live, SingleName), directory: false); + Write(Path.Combine(liveDirectory, ManifestName), directory: true); + + Assert.That(File.ReadAllBytes(Path.Combine(Vectors(), SingleName)), Is.EqualTo(File.ReadAllBytes(Path.Combine(live, SingleName))), SingleName); + + var committed = Tree(Path.Combine(Vectors(), DirectoryName)); + var written = Tree(liveDirectory); + Assert.That(committed.Select(entry => entry.Path), Is.EqualTo(written.Select(entry => entry.Path)), DirectoryName); + for (var i = 0; i < committed.Length; i++) + Assert.That(committed[i].Bytes, Is.EqualTo(written[i].Bytes), committed[i].Path); + } + } +} diff --git a/tests/GameCult.Caching.Tests/CachelessSingleFileTests.cs b/tests/GameCult.Caching.Tests/CachelessSingleFileTests.cs new file mode 100644 index 00000000..43754142 --- /dev/null +++ b/tests/GameCult.Caching.Tests/CachelessSingleFileTests.cs @@ -0,0 +1,1530 @@ +#nullable enable +using System; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using GameCult.Caching.MessagePack; +using MessagePack; +using NUnit.Framework; +using static GameCult.Caching.Tests.ElementIdTests; + +namespace GameCult.Caching.Tests +{ + // A single-file store file is read and written without a cache through ReadStoreSnapshot, WriteRecord and WriteDocument: the same reader, lock, + // atomic replace and header rule a cache's store uses. CultMesh's single-file document helpers are built on these members. + public class CachelessSingleFileTests + { + private static readonly CultSchemaCatalogEntry One = new() { SchemaId = "tests.cacheless.one", SchemaName = "tests.cacheless.one" }; + private static readonly CultSchemaCatalogEntry Two = new() { SchemaId = "tests.cacheless.two", SchemaName = "tests.cacheless.two" }; + private string _directory = ""; + private string _path = ""; + + [SetUp] + public void SetUp() + { + _directory = Path.Combine(Path.GetTempPath(), "cultlib-cacheless-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(_directory); + _path = Path.Combine(_directory, "store.cc"); + } + + [TearDown] + public void TearDown() + { + if (Directory.Exists(_directory)) + Directory.Delete(_directory, true); + } + + private static CultCodec Codec(CultDocumentRegistry? registry = null) => + CultCacheMessagePack.CreateCodec(registry ?? CultDocumentRegistry.Shared); + + [CultDocument("tests.cacheless.base", "tests.cacheless.base.v1")] + [MessagePackObject] + public class DeclaredBase + { + [Key(0)] [CultName] public string Name { get; set; } = ""; + } + + [CultDocument("tests.cacheless.id_base", "tests.cacheless.id_base.v1")] + [MessagePackObject] + public class IdBase + { + [Key(0)] [CultName] public string Name { get; set; } = ""; + } + + [MessagePackObject] + public sealed class IdDerived : IdBase + { + [Key(1)] public System.Collections.Generic.List Reels { get; set; } = new(); + } + + [MessagePackObject] + public sealed class UndeclaredDerived : DeclaredBase + { + [Key(1)] public string Extra { get; set; } = ""; + } + + // Reads the store file as a SingleFileMessagePackBackingStore does, and holds the first reader inside ReadStore so a test can + // see whether a second one gets in beside it. + private sealed class GatedStore : SingleFileBackingStore + { + private int _held; + private int _inside; + private int _mostInside; + public readonly ManualResetEventSlim Entered = new(); + public readonly ManualResetEventSlim Release = new(); + public CultDocumentRegistry? Attached => AttachedRegistry; + public CultDocumentRegistry? RegistryBeforeWait; + public CultDocumentRegistry? RegistryAfterWait; + + public GatedStore(string path) : base(path) { } + + public int MostInside => Volatile.Read(ref _mostInside); + + protected override byte[] SerializeStore(string formatVersion, System.Collections.Generic.IReadOnlyList catalogEntries, System.Collections.Generic.IReadOnlyList records) => + CultDocumentMessagePackSerialization.SerializeStore(formatVersion, catalogEntries, records); + + protected override byte[] SerializeRecord(CultPersistedRecord record) => + CultDocumentMessagePackSerialization.SerializePersistedRecord(record); + + protected override byte[] SerializeCatalogEntry(CultSchemaCatalogEntry entry) => + CultDocumentMessagePackSerialization.SerializeCatalogEntry(entry); + + protected override (CultPersistedStoreSnapshot Snapshot, byte[][] CatalogBytes, byte[][] RecordBytes) ReadStore(byte[] data) + { + var inside = Interlocked.Increment(ref _inside); + int seen; + while ((seen = MostInside) < inside && Interlocked.CompareExchange(ref _mostInside, inside, seen) != seen) { } + Entered.Set(); + try + { + if (Interlocked.Exchange(ref _held, 1) == 0) + { + RegistryBeforeWait = AttachedRegistry; + Release.Wait(TimeSpan.FromSeconds(30)); + RegistryAfterWait = AttachedRegistry; + } + var store = CultDocumentMessagePackSerialization.ReadStore(data); + CultDocumentMessagePackSerialization.RequireSingleFileFormat(store.Snapshot); + return store; + } + finally + { + Interlocked.Decrement(ref _inside); + } + } + + protected override byte[] SerializePayload(object document, Type documentType) => + CultDocumentMessagePackSerialization.SerializeUntyped(document, documentType, Registry); + + protected override object DeserializePayload(Type documentType, byte[] payload) => + CultDocumentMessagePackSerialization.DeserializeUntyped(documentType, payload, Registry); + } + + // ReadStoreSnapshot takes the store's gate: while one read is inside the reader, a second does not enter beside it. + [Test] + public async Task ReadStoreSnapshotTakesTheStoresGate() + { + new SingleFileMessagePackBackingStore(_path).WriteRecord(Record("a", One, 1), One); + var store = new GatedStore(_path); + + var first = Task.Run(() => store.ReadStoreSnapshot()); + Assert.That(store.Entered.Wait(TimeSpan.FromSeconds(10)), Is.True, "the first read reached the reader"); + var second = Task.Run(() => store.ReadStoreSnapshot()); + await Task.Delay(500); + var mostInside = store.MostInside; + store.Release.Set(); + await Task.WhenAll(first, second); + + Assert.That(mostInside, Is.EqualTo(1), "a second read waits at the gate while the first is inside the reader"); + Assert.That(first.Result!.Records, Has.Length.EqualTo(1)); + Assert.That(second.Result!.Records, Has.Length.EqualTo(1)); + } + + // A call's registry is attached under the store's gate: a second call with another registry waits for the first, so the + // first sees its own registry from its start to its end. + [Test] + public async Task ACallKeepsItsRegistryWhileAnotherCallWaits() + { + var mine = CultDocumentRegistry.ForTypes(new[] { typeof(IdDeck) }); + var other = CultDocumentRegistry.ForTypes(new[] { typeof(DeclaredBase) }); + new SingleFileMessagePackBackingStore(_path).WriteRecord(Record("a", One, 1), One); + var store = new GatedStore(_path); + + var first = Task.Run(() => store.WriteDocument(new CultRecordKey("first"), new IdDeck { Name = "first" }, registry: mine)); + Assert.That(store.Entered.Wait(TimeSpan.FromSeconds(10)), Is.True, "the first call reached the reader"); + var second = Task.Run(() => store.WriteDocument(new CultRecordKey("second"), new DeclaredBase { Name = "second" }, registry: other)); + await Task.Delay(500); + store.Release.Set(); + await Task.WhenAll(first, second); + + Assert.That(store.RegistryBeforeWait, Is.SameAs(mine)); + Assert.That(store.RegistryAfterWait, Is.SameAs(mine), "the second call's registry did not replace the first's mid-call"); + Assert.That(new SingleFileMessagePackBackingStore(_path).ReadStoreSnapshot()!.Records.Select(record => record.Key), Is.EquivalentTo(new[] { "a", "first", "second" })); + } + + // An attach during a cacheless call waits for it: the call finishes on the registry it was given, the store then gains the + // cache's registry, and the cache's next flush works. + [Test] + public async Task AnAttachDuringACachelessCallWaitsForIt() + { + var mine = CultDocumentRegistry.ForTypes(new[] { typeof(IdDeck) }); + var cacheRegistry = CultDocumentRegistry.ForTypes(new[] { typeof(IdDeck) }); + new SingleFileMessagePackBackingStore(_path).WriteDocument(new CultRecordKey("a"), new IdDeck { Name = "a" }, registry: mine); + var store = new GatedStore(_path); + + var write = Task.Run(() => store.WriteDocument(new CultRecordKey("first"), new IdDeck { Name = "first" }, registry: mine)); + Assert.That(store.Entered.Wait(TimeSpan.FromSeconds(10)), Is.True, "the call reached the reader"); + using var cache = new CultCache(cacheRegistry); + var attach = Task.Run(() => cache.AddBackingStore(store)); + Assert.That(attach.Wait(TimeSpan.FromMilliseconds(500)), Is.False, "the attach waits while a cacheless call is inside the store"); + store.Release.Set(); + await write; + await attach; + + Assert.That(store.RegistryAfterWait, Is.SameAs(mine), "the call kept its own registry to its end"); + Assert.That(store.Attached, Is.SameAs(cacheRegistry), "the call's end did not detach the cache's registry"); + await cache.AddAsync(new IdDeck { Name = "b" }); + Assert.DoesNotThrow(() => cache.FlushAllBackingStores()); + } + + // A cacheless call that finds the store attached by the time it holds the gate is refused, and leaves the cache's registry. + [Test] + public void ACachelessCallOnAnAttachedStoreIsRefusedAndLeavesTheRegistry() + { + var cacheRegistry = CultDocumentRegistry.ForTypes(new[] { typeof(IdDeck) }); + var store = new GatedStore(_path); + store.Release.Set(); + using var cache = new CultCache(cacheRegistry); + cache.AddBackingStore(store); + + Assert.Throws(() => store.WriteDocument(new CultRecordKey("x"), new IdDeck { Name = "x" })); + Assert.Throws(() => store.WriteRecord(Record("a", One, 1), One)); + Assert.That(store.Attached, Is.SameAs(cacheRegistry)); + } + + // A store whose hooks fire by the role of the thread inside it, so a test can hold an attach and a cacheless call at chosen + // points: the attach inside its pull (and make that pull fail), a call inside its read or inside its payload encoding. + private sealed class RoleStore : SingleFileBackingStore + { + [ThreadStatic] public static string? Role; + public bool FailAttachPull; + public readonly ManualResetEventSlim AttachInside = new(); + public readonly ManualResetEventSlim ReleaseAttach = new(); + public readonly ManualResetEventSlim CallInside = new(); + public readonly ManualResetEventSlim ReleaseCall = new(); + public readonly ManualResetEventSlim ReaderInside = new(); + public readonly ManualResetEventSlim ReleaseReader = new(); + // How many threads are inside ReadStore at once, and the most there ever were. + private int _inside, _mostInside; + public int MostInside => Volatile.Read(ref _mostInside); + // Run by the thread whose role is "hook", inside ReadStore: user code that runs under the store's gate. + [ThreadStatic] public static Action? RoleHook; + public CultDocumentRegistry? Attached => AttachedRegistry; + public bool IsAttachedToACache => Cache != null; + + public RoleStore(string path) : base(path) { } + + protected override byte[] SerializeStore(string formatVersion, System.Collections.Generic.IReadOnlyList catalogEntries, System.Collections.Generic.IReadOnlyList records) => + CultDocumentMessagePackSerialization.SerializeStore(formatVersion, catalogEntries, records); + + protected override byte[] SerializeRecord(CultPersistedRecord record) => + CultDocumentMessagePackSerialization.SerializePersistedRecord(record); + + protected override byte[] SerializeCatalogEntry(CultSchemaCatalogEntry entry) => + CultDocumentMessagePackSerialization.SerializeCatalogEntry(entry); + + protected override (CultPersistedStoreSnapshot Snapshot, byte[][] CatalogBytes, byte[][] RecordBytes) ReadStore(byte[] data) + { + var now = Interlocked.Increment(ref _inside); + int seen; + while ((seen = Volatile.Read(ref _mostInside)) < now && Interlocked.CompareExchange(ref _mostInside, now, seen) != seen) { } + try + { + if (Role == "attach") + { + AttachInside.Set(); + ReleaseAttach.Wait(TimeSpan.FromSeconds(30)); + if (FailAttachPull) + throw new InvalidOperationException("the attach's pull fails"); + } + else if (Role == "call-in-read") + { + CallInside.Set(); + ReleaseCall.Wait(TimeSpan.FromSeconds(30)); + } + else if (Role == "reader") + { + ReaderInside.Set(); + ReleaseReader.Wait(TimeSpan.FromSeconds(30)); + } + else if (Role == "hook") + { + RoleHook?.Invoke(); + } + var store = CultDocumentMessagePackSerialization.ReadStore(data); + CultDocumentMessagePackSerialization.RequireSingleFileFormat(store.Snapshot); + return store; + } + finally + { + Interlocked.Decrement(ref _inside); + } + } + + protected override byte[] SerializePayload(object document, Type documentType) + { + if (Role == "call-in-payload") + { + CallInside.Set(); + ReleaseCall.Wait(TimeSpan.FromSeconds(30)); + } + return CultDocumentMessagePackSerialization.SerializeUntyped(document, documentType, Registry); + } + + protected override object DeserializePayload(Type documentType, byte[] payload) => + CultDocumentMessagePackSerialization.DeserializeUntyped(documentType, payload, Registry); + } + + // Runs body on its own thread under a role, and returns the thread with whatever it threw. + private static (Thread Thread, Func Error) OnThread(string role, Action body) + { + Exception? error = null; + var thread = new Thread(() => + { + RoleStore.Role = role; + try { body(); } + catch (Exception e) { error = e; } + finally { RoleStore.Role = null; } + }) { IsBackground = true }; + thread.Start(); + return (thread, () => error); + } + + private static void WaitUntilBlocked(Thread thread) => + Assert.That(SpinWait.SpinUntil(() => thread.ThreadState.HasFlag(ThreadState.WaitSleepJoin), TimeSpan.FromSeconds(10)), Is.True, "the thread is blocked at a gate"); + + // A cacheless call that arrives during an attach waits for the attach to end, whatever the attach does. The call's thread blocks + // on the store's gate (it does not read Cache first and queue on the cache's gate), so when the pull fails the call runs, and the + // next attach waits for it; when the pull succeeds the call finds the cache and is refused, leaving the cache's registry. + [TestCase(true)] + [TestCase(false)] + public async Task ACallArrivingDuringAnAttachWaitsForTheAttachToEnd(bool theAttachFails) + { + var callRegistry = CultDocumentRegistry.ForTypes(new[] { typeof(IdDeck) }); + var registry1 = CultDocumentRegistry.ForTypes(new[] { typeof(IdDeck) }); + var registry2 = CultDocumentRegistry.ForTypes(new[] { typeof(IdDeck) }); + new SingleFileMessagePackBackingStore(_path).WriteDocument(new CultRecordKey("a"), new IdDeck { Name = "a" }, registry: callRegistry); + var store = new RoleStore(_path) { FailAttachPull = theAttachFails }; + using var cache1 = new CultCache(registry1); + var attach = OnThread("attach", () => cache1.AddBackingStore(store)); + Assert.That(store.AttachInside.Wait(TimeSpan.FromSeconds(10)), Is.True, "the attach reached its pull"); + var call = OnThread("call-in-read", () => store.WriteDocument(new CultRecordKey("first"), new IdDeck { Name = "first" }, registry: callRegistry)); + WaitUntilBlocked(call.Thread); + Assert.That(store.CallInside.IsSet, Is.False, "the call waits while the attach is inside the store"); + + store.ReleaseAttach.Set(); + attach.Thread.Join(); + Assert.That(attach.Error(), theAttachFails ? Is.InstanceOf() : Is.Null); + + if (!theAttachFails) + { + call.Thread.Join(); + Assert.That(call.Error(), Is.InstanceOf(), "the attach succeeded, so the call is refused"); + Assert.That(store.CallInside.IsSet, Is.False); + Assert.That(store.Attached, Is.SameAs(registry1), "the refused call left the cache's registry"); + await cache1.AddAsync(new IdDeck { Name = "b" }); + Assert.DoesNotThrow(() => cache1.FlushAllBackingStores()); + return; + } + + Assert.That(store.CallInside.Wait(TimeSpan.FromSeconds(10)), Is.True, "the attach failed, so the call runs"); + using var cache2 = new CultCache(registry2); + var secondAttach = Task.Run(() => cache2.AddBackingStore(store)); + Assert.That(secondAttach.Wait(TimeSpan.FromMilliseconds(500)), Is.False, "the next attach waits while the call is inside the store"); + store.ReleaseCall.Set(); + call.Thread.Join(); + await secondAttach; + + Assert.That(call.Error(), Is.Null); + Assert.That(store.Attached, Is.SameAs(registry2), "the call's end did not detach the second cache's registry"); + await cache2.AddAsync(new IdDeck { Name = "c" }); + Assert.DoesNotThrow(() => cache2.FlushAllBackingStores()); + var keys = new SingleFileMessagePackBackingStore(_path).ReadStoreSnapshot()!.Records.Select(record => record.Key).ToArray(); + Assert.That(keys, Has.Length.EqualTo(3), "the seed, the call's record and the cache's record"); + Assert.That(keys, Does.Contain("a").And.Contain("first")); + } + + // The attach waits for a call that is already inside the store, wherever in the call it is: here inside the payload encoding, + // before the file is read. The call ends on its own registry, then the cache owns the store and its registry. + [Test] + public async Task AnAttachDuringACallEncodingItsPayloadWaitsForIt() + { + var callRegistry = CultDocumentRegistry.ForTypes(new[] { typeof(IdDeck) }); + var cacheRegistry = CultDocumentRegistry.ForTypes(new[] { typeof(IdDeck) }); + var store = new RoleStore(_path); + var call = OnThread("call-in-payload", () => store.WriteDocument(new CultRecordKey("first"), new IdDeck { Name = "first" }, registry: callRegistry)); + Assert.That(store.CallInside.Wait(TimeSpan.FromSeconds(10)), Is.True, "the call reached its payload encoding"); + using var cache = new CultCache(cacheRegistry); + var attach = Task.Run(() => cache.AddBackingStore(store)); + Assert.That(attach.Wait(TimeSpan.FromMilliseconds(500)), Is.False, "the attach waits while a call is inside the store"); + + store.ReleaseCall.Set(); + call.Thread.Join(); + await attach; + + Assert.That(call.Error(), Is.Null); + Assert.That(store.Attached, Is.SameAs(cacheRegistry), "the call's end did not detach the cache's registry"); + await cache.AddAsync(new IdDeck { Name = "b" }); + Assert.DoesNotThrow(() => cache.FlushAllBackingStores()); + } + + // An attach the cache refuses before it pulls leaves the store as it found it: no cache, no registry, and a cacheless call + // still runs. + [Test] + public void ARefusedAttachLeavesTheStoreCacheless() + { + var cacheRegistry = CultDocumentRegistry.ForTypes(new[] { typeof(IdDeck) }); + using var cache = new CultCache(cacheRegistry); + cache.AddBackingStore(new RoleStore(Path.Combine(_directory, "first.cc"))); + var second = new RoleStore(_path); + + Assert.Throws(() => cache.AddBackingStore(second), "a second untyped store is refused"); + Assert.DoesNotThrow(() => second.WriteDocument(new CultRecordKey("a"), new IdDeck { Name = "a" }, registry: cacheRegistry)); + + Assert.That(second.Attached, Is.Null); + Assert.That(second.ReadStoreSnapshot()!.Records, Has.Length.EqualTo(1)); + } + + // WriteDocument stores the document as T: the schema id, the id plan, the header and the bytes all come from the one + // declared type, as they did when CultMesh encoded typeof(T). + [Test] + public void WriteDocumentEncodesTheDeclaredTypeNotTheRuntimeType() + { + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(DeclaredBase) }); + + new SingleFileMessagePackBackingStore(_path).WriteDocument(new CultRecordKey("d"), new UndeclaredDerived { Name = "n", Extra = "x" }, registry: registry); + + var record = new SingleFileMessagePackBackingStore(_path).ReadStoreSnapshot()!.Records.Single(); + var asBase = CultDocumentMessagePackSerialization.SerializeUntyped(new DeclaredBase { Name = "n" }, typeof(DeclaredBase), registry); + Assert.That(record.SchemaId, Is.EqualTo(registry.GetRequired().SchemaId)); + Assert.That(record.Payload, Is.EqualTo(asBase), "base bytes under the base schema: one slot, not the derived type's two"); + } + + // The header, like the bytes and the id plan, comes from the declared type: ids that only the derived type holds are not the + // stored document's. + [Test] + public void WriteDocumentMarksTheStoreByTheDeclaredTypeNotTheRuntimeType() + { + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(IdBase) }); + var derived = new IdDerived { Name = "n", Reels = { new IdReel { Id = "0123456789ab", Label = "r" } } }; + + new SingleFileMessagePackBackingStore(_path).WriteDocument(new CultRecordKey("d"), derived, registry: registry); + + var snapshot = new SingleFileMessagePackBackingStore(_path).ReadStoreSnapshot()!; + Assert.That(snapshot.FormatVersion, Is.EqualTo(CultPersistedStoreSnapshot.FormatV1), "the stored IdBase holds no element id"); + Assert.That(snapshot.Records.Single().Payload, Is.EqualTo(CultDocumentMessagePackSerialization.SerializeUntyped(new IdBase { Name = "n" }, typeof(IdBase), registry))); + } + + // The id plan comes from the declared type too: an unset id that only the derived type holds is not the stored document's, so + // it is not minted into the caller's object. + [Test] + public void WriteDocumentMintsIdsOfTheDeclaredTypeNotTheRuntimeType() + { + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(IdBase) }); + var derived = new IdDerived { Name = "n", Reels = { new IdReel { Label = "r" } } }; + + new SingleFileMessagePackBackingStore(_path).WriteDocument(new CultRecordKey("d"), derived, registry: registry); + + Assert.That(derived.Reels.Single().Id, Is.Null.Or.Empty, "an id outside the declared type is left unset"); + } + + // No call leaves its registry on the store: after a write, a refused write and WriteRecord, the store holds none. + [Test] + public void NoCacheCallLeavesARegistryOnTheStore() + { + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(IdDeck) }); + new SingleFileMessagePackBackingStore(_path).WriteRecord(Record("a", One, 1), One); + var store = new GatedStore(_path); + store.Release.Set(); + + store.WriteDocument(new CultRecordKey("d"), new IdDeck { Name = "d" }, registry: registry); + Assert.That(store.Attached, Is.Null, "after WriteDocument"); + store.WriteRecord(Record("b", One, 2), One, holdsIds: false, registry: registry); + Assert.That(store.Attached, Is.Null, "after WriteRecord"); + + File.WriteAllBytes(_path, new byte[] { 0xC1 }); + Assert.Throws(() => store.WriteDocument(new CultRecordKey("e"), new IdDeck { Name = "e" }, registry: registry)); + Assert.That(store.Attached, Is.Null, "after a refused write"); + } + + // The registry is the call's argument. A typed call on a store instance leaves nothing behind, so a later WriteRecord that + // gives none decides the header as a fresh instance does. + [Test] + public async Task ATypedCallLeavesNoRegistryOnTheStore() + { + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(IdDeck) }); + using (var cache = new CultCache(registry)) + { + cache.AddBackingStore(new SingleFileMessagePackBackingStore(_path)); + await cache.AddAsync(new IdDeck { Name = "deck", Reels = { new IdReel { Label = "reel" } } }); + cache.FlushAllBackingStores(); + } + void Unmark() + { + var written = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(_path)); + written.FormatVersion = CultPersistedStoreSnapshot.FormatV1; + File.WriteAllBytes(_path, CultDocumentMessagePackSerialization.SerializeSnapshot(written)); + } + Unmark(); + var store = new SingleFileMessagePackBackingStore(_path); + + store.WriteDocument(new CultRecordKey("again"), new IdDeck { Name = "again" }, registry: registry); + Assert.That(store.ReadStoreSnapshot()!.FormatVersion, Is.EqualTo(CultPersistedStoreSnapshot.FormatV3), "the typed call, given a registry, read the copied ids"); + Unmark(); + store.WriteRecord(Record("other", One, 1), One, holdsIds: false, registry: null); + + Assert.That(store.ReadStoreSnapshot()!.FormatVersion, Is.EqualTo(CultPersistedStoreSnapshot.FormatV1), "a write given no registry decides nothing for copied records"); + } + + private static CultPersistedRecord Record(string key, CultSchemaCatalogEntry entry, params byte[] payload) => + new() { Key = key, SchemaId = entry.SchemaId, StoredAt = "2026-10-22T00:00:00.0000000+00:00", Payload = payload }; + + [Test] + public void WriteRecordKeepsAnotherWritersRecord() + { + new SingleFileMessagePackBackingStore(_path).WriteRecord(Record("a", One, 1), One); + new SingleFileMessagePackBackingStore(_path).WriteRecord(Record("b", Two, 2), Two); + new SingleFileMessagePackBackingStore(_path).WriteRecord(Record("a", One, 3), One); + + var snapshot = new SingleFileMessagePackBackingStore(_path).ReadStoreSnapshot()!; + + Assert.That(snapshot.Records, Has.Length.EqualTo(2)); + Assert.That(Array.Find(snapshot.Records, record => record.Key == "a")!.Payload, Is.EqualTo(new byte[] { 3 })); + Assert.That(Array.Find(snapshot.Records, record => record.Key == "b")!.Payload, Is.EqualTo(new byte[] { 2 })); + Assert.That(snapshot.SchemaCatalog, Has.Length.EqualTo(2)); + } + + [Test] + public void WriteRecordRefusesOnAStoreAttachedToACache() + { + new SingleFileMessagePackBackingStore(_path).WriteRecord(Record("a", One, 1), One); + var before = File.ReadAllBytes(_path); + var store = new SingleFileMessagePackBackingStore(_path); + using var cache = new CultCache(); + cache.AddBackingStore(store); + + var error = Assert.Throws(() => store.WriteRecord(Record("b", Two, 2), Two))!; + + Assert.That(error.Message, Does.Contain("write through the cache")); + Assert.That(File.ReadAllBytes(_path), Is.EqualTo(before)); + } + + [Test] + public void ReadStoreSnapshotOfNothingIsNull() + { + Assert.That(new SingleFileMessagePackBackingStore(_path).ReadStoreSnapshot(), Is.Null); + } + + [Test] + public void ReadStoreSnapshotRefusesAV4Stamp() + { + new SingleFileMessagePackBackingStore(_path).WriteRecord(Record("a", One, 1), One); + var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(_path)); + snapshot.FormatVersion = "cultcache.store.v4"; + File.WriteAllBytes(_path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + + var error = Assert.Throws(() => new SingleFileMessagePackBackingStore(_path).ReadStoreSnapshot())!; + + Assert.That(error.Message, Does.Contain("cultcache.store.v4")); + } + + [Test] + public void WriteRecordWithIdsMarksV3() + { + new SingleFileMessagePackBackingStore(_path).WriteRecord(Record("a", One, 1), One); + Assert.That(new SingleFileMessagePackBackingStore(_path).ReadStoreSnapshot()!.FormatVersion, Is.Not.EqualTo(CultPersistedStoreSnapshot.FormatV3)); + + new SingleFileMessagePackBackingStore(_path).WriteRecord(Record("b", Two, 2), Two, holdsIds: true); + + Assert.That(new SingleFileMessagePackBackingStore(_path).ReadStoreSnapshot()!.FormatVersion, Is.EqualTo(CultPersistedStoreSnapshot.FormatV3)); + } + + [Test] + public void WriteRecordRefusesNullArguments() + { + var store = new SingleFileMessagePackBackingStore(_path); + + Assert.Throws(() => store.WriteRecord(null!, One)); + Assert.Throws(() => store.WriteRecord(Record("a", One, 1), null!)); + Assert.That(File.Exists(_path), Is.False); + } + + [Test] + public void ARefusedWriteDocumentOnAnAttachedStoreNeverReachesTheRegistryItWasGiven() + { + var store = new SingleFileMessagePackBackingStore(_path); + using var cache = new CultCache(CultDocumentRegistry.ForTypes(new[] { typeof(IdDeck) })); + cache.AddBackingStore(store); + var elsewhere = CultDocumentRegistry.ForTypes(new[] { typeof(StoreRoutingTests.RoutingNote) }); + + var error = Assert.Throws(() => store.WriteDocument(new CultRecordKey("deck"), new IdDeck { Name = "d" }, registry: elsewhere))!; + + Assert.That(error.Message, Does.Contain("write through the cache")); + } + + [Test] + public void WriteDocumentRefusesANullDocumentAndDecodeDocumentANullPayload() + { + var store = new SingleFileMessagePackBackingStore(_path); + Assert.Throws(() => store.WriteDocument(new CultRecordKey("deck"), null!)); + Assert.Throws(() => Codec().DecodeDocument(new CultRecordKey("deck"), null!)); + } + + [Test] + public void WriteDocumentStoresTheGivenStoredAtAndOtherwiseNow() + { + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(IdDeck) }); + var store = new SingleFileMessagePackBackingStore(_path); + store.WriteDocument(new CultRecordKey("fixed"), new IdDeck { Name = "a" }, "2026-01-02T03:04:05.0000000+00:00", registry); + store.WriteDocument(new CultRecordKey("now"), new IdDeck { Name = "b" }, null, registry); + store.WriteDocument(new CultRecordKey("blank"), new IdDeck { Name = "c" }, " ", registry); + + var records = new SingleFileMessagePackBackingStore(_path).ReadStoreSnapshot()!.Records.ToDictionary(record => record.Key); + + Assert.That(records["fixed"].StoredAt, Is.EqualTo("2026-01-02T03:04:05.0000000+00:00")); + Assert.That(records["now"].StoredAt, Does.Match(@"^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d{7}\+00:00$"), "round-trip format"); + Assert.That(DateTimeOffset.Parse(records["now"].StoredAt), Is.EqualTo(DateTimeOffset.UtcNow).Within(TimeSpan.FromMinutes(5))); + Assert.That(DateTimeOffset.Parse(records["blank"].StoredAt), Is.EqualTo(DateTimeOffset.UtcNow).Within(TimeSpan.FromMinutes(5))); + } + + [Test] + public void WriteDocumentMintsRandomIdsAndDecodeDocumentMintsTheSameIdsEveryTime() + { + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(IdDeck) }); + string Written(string name) + { + var path = Path.Combine(_directory, name); + var deck = new IdDeck { Name = "deck", Reels = { new IdReel { Label = "reel" } } }; + new SingleFileMessagePackBackingStore(path).WriteDocument(new CultRecordKey("deck"), deck, registry: registry); + return deck.Reels[0].Id; + } + Assert.That(Written("one.cc"), Is.Not.EqualTo(Written("two.cc")), "a write mints random ids"); + + var unset = CultDocumentMessagePackSerialization.SerializeUntyped(new IdDeck { Name = "deck", Reels = { new IdReel { Label = "reel" } } }, typeof(IdDeck), registry); + var first = Codec(registry).DecodeDocument(new CultRecordKey("deck"), unset).Reels[0].Id; + var second = Codec(registry).DecodeDocument(new CultRecordKey("deck"), unset).Reels[0].Id; + Assert.That(first, Does.Match("^[0-9a-f]{12}$")); + Assert.That(second, Is.EqualTo(first), "a load mints from the key and path, so every reader mints the same id"); + } + + // The write is under the store's lock and waits for it: a write that finds the lock held lands after it is released, and + // never without it. + [Test] + public async Task WriteRecordWaitsForTheStoreLock() + { + var write = default(Task); + using (new FileStream(_path + ".lock", FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None)) + { + write = Task.Run(() => new SingleFileMessagePackBackingStore(_path).WriteRecord(Record("a", One, 1), One)); + await Task.Delay(500); + Assert.That(write.IsCompleted, Is.False, "the write waits while another holder has the lock"); + Assert.That(File.Exists(_path), Is.False); + } + + await write; + + Assert.That(new SingleFileMessagePackBackingStore(_path).ReadStoreSnapshot()!.Records, Has.Length.EqualTo(1)); + } + + // The registry the payload was encoded with lets the write decide the header for the records it copies: another runtime may + // have written ids under an unmarked header, and a write that can read them marks the store. Without a registry it decides + // nothing for them. + [TestCase(true, CultPersistedStoreSnapshot.FormatV3)] + [TestCase(false, CultPersistedStoreSnapshot.FormatV1)] + public async Task WriteRecordGivenARegistryMarksAStoreWhoseCopiedRecordHoldsIds(bool withRegistry, string expected) + { + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(IdDeck) }); + using (var cache = new CultCache(registry)) + { + cache.AddBackingStore(new SingleFileMessagePackBackingStore(_path)); + await cache.AddAsync(new IdDeck { Name = "deck", Reels = { new IdReel { Label = "reel" } } }); + cache.FlushAllBackingStores(); + } + + var written = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(_path)); + Assert.That(written.FormatVersion, Is.EqualTo(CultPersistedStoreSnapshot.FormatV3), "the cache marked the store it wrote ids into"); + written.FormatVersion = CultPersistedStoreSnapshot.FormatV1; + File.WriteAllBytes(_path, CultDocumentMessagePackSerialization.SerializeSnapshot(written)); + + new SingleFileMessagePackBackingStore(_path).WriteRecord(Record("other", One, 1), One, holdsIds: false, registry: withRegistry ? registry : null); + + Assert.That(new SingleFileMessagePackBackingStore(_path).ReadStoreSnapshot()!.FormatVersion, Is.EqualTo(expected)); + } + + [Test] + public void WriteDocumentMintsIdsMarksV3AndKeepsTheOtherRecord() + { + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(IdDeck) }); + new SingleFileMessagePackBackingStore(_path).WriteRecord(Record("other", One, 1), One); + var deck = new IdDeck { Name = "deck", Reels = { new IdReel { Label = "reel" } } }; + + new SingleFileMessagePackBackingStore(_path).WriteDocument(new CultRecordKey("deck"), deck, registry: registry); + + var snapshot = new SingleFileMessagePackBackingStore(_path).ReadStoreSnapshot()!; + Assert.That(snapshot.FormatVersion, Is.EqualTo(CultPersistedStoreSnapshot.FormatV3)); + Assert.That(snapshot.Records.Select(record => record.Key), Is.EquivalentTo(new[] { "other", "deck" })); + Assert.That(deck.Reels[0].Id, Does.Match("^[0-9a-f]{12}$"), "the id the file holds was minted on the document"); + var read = Codec(registry).DecodeDocument(new CultRecordKey("deck"), snapshot.Records.Single(record => record.Key == "deck").Payload); + Assert.That(read.Reels[0].Id, Is.EqualTo(deck.Reels[0].Id)); + } + + [Test] + public void WriteDocumentGivesMintedIdsBackWhenTheWriteIsRefused() + { + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(IdDeck) }); + File.WriteAllBytes(_path, new byte[] { 0xC1 }); + var deck = new IdDeck { Name = "deck", Reels = { new IdReel { Label = "reel" } } }; + + Assert.Throws(() => + new SingleFileMessagePackBackingStore(_path).WriteDocument(new CultRecordKey("deck"), deck, registry: registry)); + + Assert.That(deck.Reels[0].Id, Is.Null.Or.Empty, "a refused write leaves the document as it was"); + } + + [Test] + public void WriteDocumentRefusesOnAStoreAttachedToACache() + { + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(IdDeck) }); + var store = new SingleFileMessagePackBackingStore(_path); + using var cache = new CultCache(registry); + cache.AddBackingStore(store); + + Assert.That(Assert.Throws(() => store.WriteDocument(new CultRecordKey("deck"), new IdDeck { Name = "d" }))!.Message, Does.Contain("write through the cache")); + Assert.That(File.Exists(_path), Is.False); + } + + // A document whose getter runs user code while MessagePack encodes it, under the store's gate. + [CultDocument("tests.cacheless.tap", "tests.cacheless.tap.v1")] + [MessagePackObject] + public class TapDoc + { + [ThreadStatic] public static Action? Hook; + [Key(0)] [CultName] public string Name { get; set; } = ""; + [Key(1)] public string Tap { get { var hook = Hook; Hook = null; hook?.Invoke(); return "t"; } set { } } + [Key(2)] public System.Collections.Generic.List Reels { get; set; } = new(); + } + + private static void Finishes(Thread thread, string what) => + Assert.That(thread.Join(TimeSpan.FromSeconds(10)), Is.True, what + " finished (a deadlock fails here, it does not hang)"); + + private static bool Refused(Exception? error) + { + for (var inner = error; inner != null; inner = inner.InnerException) + if (inner is InvalidOperationException) + return true; + return false; + } + + // While a thread holds a store's gate (a cacheless call, an attach's pull) it enters no cache it does not already hold, reads + // included, and no second store. The call is refused where the getter ran, the file is not written, and minted ids are given back. + [Test] + public void AGetterThatEntersACacheDuringACachelessWriteIsRefused() + { + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(TapDoc) }); + using var unrelated = new CultCache(registry); + var doc = new TapDoc { Name = "d", Reels = { new IdReel() } }; + TapDoc.Hook = () => unrelated.Get(new CultRecordKey("x")); + + var error = Assert.Catch(() => new SingleFileMessagePackBackingStore(_path).WriteDocument(new CultRecordKey("d"), doc, registry: registry)); + + Assert.That(Refused(error), Is.True, "the getter's read of a cache was refused"); + Assert.That(File.Exists(_path), Is.False); + Assert.That(doc.Reels[0].Id, Is.Null.Or.Empty, "the minted id was given back"); + } + + [Test] + public void AGetterThatWritesACacheDuringACachelessWriteIsRefused() + { + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(TapDoc) }); + using var unrelated = new CultCache(registry); + TapDoc.Hook = () => _ = unrelated.AddAsync(new TapDoc { Name = "other" }); + + var error = Assert.Catch(() => new SingleFileMessagePackBackingStore(_path).WriteDocument(new CultRecordKey("d"), new TapDoc { Name = "d" }, registry: registry)); + + Assert.That(Refused(error), Is.True, "the getter's write to a cache was refused"); + Assert.That(File.Exists(_path), Is.False); + Assert.That(unrelated.AllEntries, Is.Empty, "the refused write reached nothing"); + } + + // Soul P4: the writer holds the store's gate and its getter reads the cache that another thread is attaching the same store to. + // The attach holds the cache's gate and waits for the store's. The getter is refused instead of waiting on the cache's gate. + [Test] + public async Task AGetterThatEntersTheAttachingCacheIsRefusedNotDeadlocked() + { + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(TapDoc) }); + var store = new RoleStore(_path); + var cache = new CultCache(registry); + var inside = new ManualResetEventSlim(); + var attachWaits = new ManualResetEventSlim(); + var write = OnThread("writer", () => + { + TapDoc.Hook = () => + { + inside.Set(); + attachWaits.Wait(TimeSpan.FromSeconds(30)); + _ = cache.Get(new CultRecordKey("anything")); + }; + store.WriteDocument(new CultRecordKey("d"), new TapDoc { Name = "d" }, registry: registry); + }); + Assert.That(inside.Wait(TimeSpan.FromSeconds(10)), Is.True, "the getter ran under the store's gate"); + var attach = OnThread("attach-after", () => cache.AddBackingStore(store)); + WaitUntilBlocked(attach.Thread); + attachWaits.Set(); + + Finishes(write.Thread, "the writer"); + Finishes(attach.Thread, "the attach"); + Assert.That(Refused(write.Error()), Is.True, "the write was refused"); + Assert.That(attach.Error(), Is.Null); + await cache.AddAsync(new TapDoc { Name = "after" }); + Assert.DoesNotThrow(() => cache.FlushAllBackingStores()); + // Disposed only after every join passed: a deadlocked gate would make Dispose wait for it. + cache.Dispose(); + } + + // The attach holds the store's gate while it pulls; user code in the pull may not enter a second store. + [Test] + public void AnAttachPullThatEntersAnotherStoreIsRefused() + { + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(IdDeck) }); + new SingleFileMessagePackBackingStore(_path).WriteDocument(new CultRecordKey("a"), new IdDeck { Name = "a" }, registry: registry); + var store = new RoleStore(_path); + var second = new SingleFileMessagePackBackingStore(Path.Combine(_directory, "second.cc")); + var cache = new CultCache(registry); + var attach = OnThread("hook", () => + { + RoleStore.RoleHook = () => second.WriteDocument(new CultRecordKey("b"), new IdDeck { Name = "b" }, registry: registry); + try { cache.AddBackingStore(store); } + finally { RoleStore.RoleHook = null; } + }); + Finishes(attach.Thread, "the attach"); + + Assert.That(Refused(attach.Error()), Is.True, "the pull's write to a second store was refused"); + var refusal = attach.Error()!; + while (refusal is not InvalidOperationException && refusal.InnerException != null) + refusal = refusal.InnerException; + while (refusal.InnerException is InvalidOperationException deeper) + refusal = deeper; + Assert.That(refusal.Message, Does.StartWith("A store's gate was entered inside another store's hold")); + Assert.That(File.Exists(Path.Combine(_directory, "second.cc")), Is.False); + Assert.That(cache.BackingStores, Is.Empty, "the failed attach left nothing behind"); + Assert.DoesNotThrow(() => store.WriteDocument(new CultRecordKey("c"), new IdDeck { Name = "c" }, registry: registry), "the store is cacheless after"); + // Disposed only after every join passed: a deadlocked gate would make Dispose wait for it. + cache.Dispose(); + } + + // Soul P4's window: a reader queued on a failed attach's cache gate must retry under the store's gate. Whichever of the reader and + // the cacheless writer queued behind the attach gets the store first is alone inside it, and a second cache's attach waits for it. + [TestCase("snapshot")] + [TestCase("pullall")] + public async Task AReaderQueuedOnAFailedAttachRunsUnderTheStoresGate(string how) + { + var callRegistry = CultDocumentRegistry.ForTypes(new[] { typeof(IdDeck) }); + var registry1 = CultDocumentRegistry.ForTypes(new[] { typeof(IdDeck) }); + var registry2 = CultDocumentRegistry.ForTypes(new[] { typeof(IdDeck) }); + new SingleFileMessagePackBackingStore(_path).WriteDocument(new CultRecordKey("a"), new IdDeck { Name = "a" }, registry: callRegistry); + var store = new RoleStore(_path) { FailAttachPull = true }; + var cache1 = new CultCache(registry1); + var cache2 = new CultCache(registry2); + var attach1 = OnThread("attach", () => cache1.AddBackingStore(store)); + Assert.That(store.AttachInside.Wait(TimeSpan.FromSeconds(10)), Is.True); + CultPersistedStoreSnapshot? snapshot = null; + var reader = OnThread("reader", () => { if (how == "snapshot") snapshot = store.ReadStoreSnapshot(); else store.PullAll(); }); + WaitUntilBlocked(reader.Thread); + var writer = OnThread("call-in-read", () => store.WriteDocument(new CultRecordKey("first"), new IdDeck { Name = "first" }, registry: callRegistry)); + WaitUntilBlocked(writer.Thread); + + store.ReleaseAttach.Set(); + Finishes(attach1.Thread, "the failed attach"); + Assert.That(SpinWait.SpinUntil(() => store.ReaderInside.IsSet || store.CallInside.IsSet, TimeSpan.FromSeconds(10)), Is.True, "one of them is inside the store"); + var attach2 = OnThread("attach2", () => cache2.AddBackingStore(store)); + WaitUntilBlocked(attach2.Thread); + Assert.That(attach2.Thread.IsAlive, Is.True, "the second cache's attach waits while a reader or a call is inside the store"); + + store.ReleaseReader.Set(); + store.ReleaseCall.Set(); + Finishes(reader.Thread, "the reader"); + Finishes(writer.Thread, "the writer"); + Finishes(attach2.Thread, "the second attach"); + Assert.That(store.MostInside, Is.EqualTo(1), "never two threads inside the store's read at once"); + // A PullAll on a cacheless store with no registry (the cacheless writer that ran before it detached its own) is a usage error; + // being refused for entering a cache's hold is not. + Assert.That(reader.Error()?.Message, Is.Null.Or.Contain("not attached to a CultDocumentRegistry"), "the reader was not refused"); + if (how == "snapshot") + Assert.That(snapshot?.Records.Select(record => record.Key), Does.Contain("a"), "the retried read ran and returned the file's snapshot"); + Assert.That(attach2.Error(), Is.Null); + await cache2.AddAsync(new IdDeck { Name = "b" }); + Assert.DoesNotThrow(() => cache2.FlushAllBackingStores()); + // Disposed only after every join passed: a deadlocked gate would make Dispose wait for it. + cache1.Dispose(); + cache2.Dispose(); + } + + // After a failed attach left the store cacheless, the next cache attaches it, and a PullAll that queues on that attach (it guesses + // the second cache from Cache) runs under the second cache's gate once the attach ends: no refusal, and the cache holds the records. + [Test] + public void AFailedAttachThenPullAllLoadsIntoTheSecondCache() + { + var callRegistry = CultDocumentRegistry.ForTypes(new[] { typeof(IdDeck) }); + new SingleFileMessagePackBackingStore(_path).WriteDocument(new CultRecordKey("a"), new IdDeck { Name = "a" }, registry: callRegistry); + var store = new RoleStore(_path) { FailAttachPull = true }; + var cache1 = new CultCache(CultDocumentRegistry.ForTypes(new[] { typeof(IdDeck) })); + var cache2 = new CultCache(CultDocumentRegistry.ForTypes(new[] { typeof(IdDeck) })); + var attach1 = OnThread("attach", () => cache1.AddBackingStore(store)); + Assert.That(store.AttachInside.Wait(TimeSpan.FromSeconds(10)), Is.True); + store.ReleaseAttach.Set(); + Finishes(attach1.Thread, "the failed attach"); + Assert.That(Refused(attach1.Error()), Is.True); + + store.AttachInside.Reset(); + store.ReleaseAttach.Reset(); + store.FailAttachPull = false; + var attach2 = OnThread("attach", () => cache2.AddBackingStore(store)); + Assert.That(store.AttachInside.Wait(TimeSpan.FromSeconds(10)), Is.True, "the second attach is inside its pull"); + var pull = OnThread("pull", () => store.PullAll()); + WaitUntilBlocked(pull.Thread); + + store.ReleaseAttach.Set(); + Finishes(attach2.Thread, "the second attach"); + Finishes(pull.Thread, "the queued PullAll"); + Assert.That(attach2.Error(), Is.Null); + Assert.That(pull.Error(), Is.Null, "the queued PullAll was not refused"); + Assert.That(cache2.Get(new CultRecordKey("a")), Is.Not.Null, "the second cache holds the file's records"); + // Disposed only after every join passed: a deadlocked gate would make Dispose wait for it. + cache1.Dispose(); + cache2.Dispose(); + } + + // A reader that guessed "no cache" and queued on the store's gate behind an attach finds the cache when it gets the gate, and runs + // under the cache's gate: while it is inside the store, the cache's own readers wait. + [Test] + public void AReaderThatQueuedBeforeAnAttachRunsUnderTheCacheGateAfterIt() + { + var callRegistry = CultDocumentRegistry.ForTypes(new[] { typeof(IdDeck) }); + new SingleFileMessagePackBackingStore(_path).WriteDocument(new CultRecordKey("a"), new IdDeck { Name = "a" }, registry: callRegistry); + var store = new RoleStore(_path); + var cache = new CultCache(CultDocumentRegistry.ForTypes(new[] { typeof(IdDeck) })); + var call = OnThread("call-in-read", () => store.WriteDocument(new CultRecordKey("first"), new IdDeck { Name = "first" }, registry: callRegistry)); + Assert.That(store.CallInside.Wait(TimeSpan.FromSeconds(10)), Is.True, "the call holds the store's gate"); + var attach = OnThread("attach", () => cache.AddBackingStore(store)); + WaitUntilBlocked(attach.Thread); + var reader = OnThread("reader", () => store.ReadStoreSnapshot()); + WaitUntilBlocked(reader.Thread); + + store.ReleaseCall.Set(); + Assert.That(store.AttachInside.Wait(TimeSpan.FromSeconds(10)), Is.True, "the attach queued first and got the store's gate first"); + store.ReleaseAttach.Set(); + Assert.That(store.ReaderInside.Wait(TimeSpan.FromSeconds(10)), Is.True, "the reader retried and is inside the store"); + var probe = OnThread("probe", () => cache.Get(new CultRecordKey("a"))); + Assert.That(SpinWait.SpinUntil(() => !probe.Thread.IsAlive || probe.Thread.ThreadState.HasFlag(ThreadState.WaitSleepJoin), TimeSpan.FromSeconds(10)), Is.True); + Assert.That(probe.Thread.IsAlive, Is.True, "the cache's own reader waits while the retried reader is inside the store: it holds the cache's gate"); + + store.ReleaseReader.Set(); + Finishes(call.Thread, "the call"); + Finishes(attach.Thread, "the attach"); + Finishes(reader.Thread, "the reader"); + Finishes(probe.Thread, "the cache's reader"); + Assert.That(attach.Error(), Is.Null); + Assert.That(reader.Error(), Is.Null); + cache.Dispose(); + } + + // The gate refuses a cache the thread does not hold; it does not refuse the cache whose gate the thread already holds. + [Test] + public void AnAttachPullMayReadTheCacheItIsAttachingTo() + { + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(IdDeck) }); + new SingleFileMessagePackBackingStore(_path).WriteDocument(new CultRecordKey("a"), new IdDeck { Name = "a" }, registry: registry); + var store = new RoleStore(_path); + var cache = new CultCache(registry); + var attach = OnThread("hook", () => + { + RoleStore.RoleHook = () => cache.Get(new CultRecordKey("a")); + try { cache.AddBackingStore(store); } + finally { RoleStore.RoleHook = null; } + }); + Finishes(attach.Thread, "the attach"); + + Assert.That(attach.Error(), Is.Null); + Assert.That(cache.BackingStores, Has.Count.EqualTo(1)); + cache.Dispose(); + } + + // A store call made from user code the same store's call is running is the same thread's own hold, not a second store. + [Test] + public void UserCodeInACachelessCallMayCallItsOwnStore() + { + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(TapDoc) }); + var store = new SingleFileMessagePackBackingStore(_path); + var descriptor = registry.GetRequired(); + TapDoc.Hook = () => store.Delete(new CultStoredDocument(new CultRecordKey("x"), DateTimeOffset.UtcNow.ToString("O"), descriptor, new TapDoc { Name = "x" })); + + Assert.DoesNotThrow(() => store.WriteDocument(new CultRecordKey("d"), new TapDoc { Name = "d" }, registry: registry)); + } + + // A failed attach leaves the store as it found it: no cache and no registry. A cacheless PullAll after it is then a usage error + // that loads nothing, so the next cache to attach admits the file's records, and creating what it cannot find does not duplicate. + [Test] + public void AFailedAttachLeavesNoRegistryAndTheNextCacheSeesTheFilesRecords() + { + var callRegistry = CultDocumentRegistry.ForTypes(new[] { typeof(IdDeck) }); + new SingleFileMessagePackBackingStore(_path).WriteDocument(new CultRecordKey("a"), new IdDeck { Name = "a" }, registry: callRegistry); + var store = new RoleStore(_path) { FailAttachPull = true }; + store.ReleaseAttach.Set(); + var cache1 = new CultCache(CultDocumentRegistry.ForTypes(new[] { typeof(IdDeck) })); + var attach1 = OnThread("attach", () => cache1.AddBackingStore(store)); + Finishes(attach1.Thread, "the failed attach"); + Assert.That(Refused(attach1.Error()), Is.True); + Assert.That(store.IsAttachedToACache, Is.False); + Assert.That(store.Attached, Is.Null, "the failed attach took its registry back"); + + Assert.That(Assert.Throws(() => store.PullAll())!.Message, Does.Contain("not attached to a CultDocumentRegistry")); + store.FailAttachPull = false; + var cache2 = new CultCache(CultDocumentRegistry.ForTypes(new[] { typeof(IdDeck) })); + var attach2 = OnThread("attach", () => cache2.AddBackingStore(store)); + Finishes(attach2.Thread, "the second attach"); + Assert.That(attach2.Error(), Is.Null); + + Assert.That(cache2.Get(new CultRecordKey("a")), Is.Not.Null, "the second cache holds the file's record"); + Assert.That(cache2.GetAll().Count(), Is.EqualTo(1)); + cache2.FlushAllBackingStores(); + Assert.That(new SingleFileMessagePackBackingStore(_path).ReadStoreSnapshot()!.Records, Has.Length.EqualTo(1)); + cache1.Dispose(); + cache2.Dispose(); + } + + // Soul P4's load: cacheless writers and snapshot readers race an attach that fails (odd rounds) or succeeds. Whoever ends up + // owning the store owns its registry, and its cache writes and flushes. A call split into a refusal hold and a body hold fails here: + // a body that runs after the attach took the store leaves the call's registry (or none) on the cache's store. + [Test] + public async Task CachelessCallsRacingAnAttachNeverLeaveTheCachesStoreWithoutItsRegistry() + { + var bad = new System.Collections.Generic.List(); + for (var round = 0; round < 60; round++) + { + var path = Path.Combine(_directory, $"load{round}.cc"); + var mine = CultDocumentRegistry.ForTypes(new[] { typeof(IdDeck) }); + var reg1 = CultDocumentRegistry.ForTypes(new[] { typeof(IdDeck) }); + var reg2 = CultDocumentRegistry.ForTypes(new[] { typeof(IdDeck) }); + var attachFails = round % 2 == 1; + var store = new RoleStore(path) { FailAttachPull = attachFails }; + store.ReleaseAttach.Set(); + store.WriteDocument(new CultRecordKey("seed"), new IdDeck { Name = "seed" }, registry: mine); + using var cache1 = new CultCache(reg1); + using var cache2 = new CultCache(reg2); + var go = new ManualResetEventSlim(); + var tasks = new System.Collections.Generic.List(); + for (var writer = 0; writer < 4; writer++) + { + var n = writer; + tasks.Add(Task.Run(() => + { + go.Wait(); + for (var k = 0; k < 8; k++) + { + try { store.WriteDocument(new CultRecordKey($"w{n}-{k}"), new IdDeck { Name = $"w{n}-{k}" }, registry: mine); } + catch (InvalidOperationException) { } + } + })); + } + for (var reader = 0; reader < 2; reader++) + tasks.Add(Task.Run(() => + { + go.Wait(); + for (var k = 0; k < 12; k++) + store.ReadStoreSnapshot(); + })); + tasks.Add(Task.Run(() => + { + go.Wait(); + Thread.SpinWait(1500 * round); + RoleStore.Role = "attach"; + try { cache1.AddBackingStore(store); } + catch (InvalidOperationException) { } + finally { RoleStore.Role = null; } + if (!store.IsAttachedToACache) + cache2.AddBackingStore(store); + })); + go.Set(); + Assert.That(Task.WhenAll(tasks).Wait(TimeSpan.FromSeconds(60)), Is.True, "the racing calls and the attach finished (a deadlock fails here, it does not hang)"); + var owner = attachFails ? cache2 : cache1; + var ownerRegistry = attachFails ? reg2 : reg1; + if (!ReferenceEquals(store.Attached, ownerRegistry)) + bad.Add($"round {round}: the owning cache's registry is not on its store"); + else + { + await owner.AddAsync(new IdDeck { Name = "after" }); + owner.FlushAllBackingStores(); + } + } + + Assert.That(bad, Is.Empty); + } + + // Every way into a cache's gate refuses a thread that holds a store's gate and not that cache's. + [TestCase("Get")] + [TestCase("GetWithSequence")] + [TestCase("GetStored")] + [TestCase("GetAll")] + [TestCase("IsDirty")] + [TestCase("BackingStores")] + [TestCase("AllEntries")] + [TestCase("AllStoredDocuments")] + [TestCase("TryGetHandle")] + public void EveryCacheGateEntryRefusesAGetterRunningUnderAStoresGate(string entry) + { + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(TapDoc) }); + using var unrelated = new CultCache(registry); + var key = new CultRecordKey("x"); + TapDoc.Hook = () => + { + switch (entry) + { + case "Get": unrelated.Get(key); break; + case "GetWithSequence": unrelated.GetWithSequence(key); break; + case "GetStored": unrelated.GetStored(key); break; + case "GetAll": unrelated.GetAll(); break; + case "IsDirty": _ = unrelated.IsDirty; break; + case "BackingStores": _ = unrelated.BackingStores; break; + case "AllEntries": _ = unrelated.AllEntries; break; + case "AllStoredDocuments": _ = unrelated.AllStoredDocuments; break; + default: unrelated.TryGetHandle(new TapDoc()); break; + } + }; + + var error = Assert.Catch(() => new SingleFileMessagePackBackingStore(_path).WriteDocument(new CultRecordKey("d"), new TapDoc { Name = "d" }, registry: registry)); + + Assert.That(Refused(error), Is.True, entry + " was refused under the store's gate"); + Assert.That(File.Exists(_path), Is.False); + } + + // Calling its own store from user code does not end the store's hold for the rest of the call: a cache entered afterwards is + // still refused. + [Test] + public void UserCodeThatCalledItsOwnStoreStillCannotEnterACacheAfterwards() + { + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(TapDoc) }); + var store = new SingleFileMessagePackBackingStore(_path); + var descriptor = registry.GetRequired(); + using var unrelated = new CultCache(registry); + TapDoc.Hook = () => + { + store.Delete(new CultStoredDocument(new CultRecordKey("x"), DateTimeOffset.UtcNow.ToString("O"), descriptor, new TapDoc { Name = "x" })); + unrelated.Get(new CultRecordKey("x")); + }; + + var error = Assert.Catch(() => store.WriteDocument(new CultRecordKey("d"), new TapDoc { Name = "d" }, registry: registry)); + + Assert.That(Refused(error), Is.True, "the read after the own-store call was refused"); + Assert.That(File.Exists(_path), Is.False); + } + + // The attach lends the pull its registry and publishes the cache before the pull: while the pull runs the store holds the loan and + // is attached, so every entry point agrees on what the store is for the whole attach. + [Test] + public void AnAttachLendsTheRegistryAndPublishesTheCacheForThePull() + { + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(IdDeck) }); + new SingleFileMessagePackBackingStore(_path).WriteDocument(new CultRecordKey("a"), new IdDeck { Name = "a" }, registry: registry); + var codec = Codec(registry); + var store = new RoleStore(_path); + using var cache = new CultCache(registry, codec); + bool cacheDuringPull = true; + CultDocumentRegistry? registryDuringPull = null; + var attach = OnThread("hook", () => + { + RoleStore.RoleHook = () => + { + cacheDuringPull = store.IsAttachedToACache; + registryDuringPull = store.Attached; + }; + try { cache.AddBackingStore(store); } + finally { RoleStore.RoleHook = null; } + }); + Finishes(attach.Thread, "the attach"); + + Assert.That(attach.Error(), Is.Null); + Assert.That(cacheDuringPull, Is.True, "the store is attached for the pull"); + Assert.That(registryDuringPull, Is.SameAs(registry)); + Assert.That(store.IsAttachedToACache, Is.True, "still attached once the pull succeeded"); + } + + // A failed attach takes back both the registry and the cache, in its own hold of the store. + [Test] + public void AFailedAttachTakesBackTheRegistryAndTheCache() + { + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(IdDeck) }); + new SingleFileMessagePackBackingStore(_path).WriteDocument(new CultRecordKey("a"), new IdDeck { Name = "a" }, registry: registry); + var store = new RoleStore(_path) { FailAttachPull = true }; + store.ReleaseAttach.Set(); + using var cache = new CultCache(registry, Codec(registry)); + var attach = OnThread("attach", () => cache.AddBackingStore(store)); + Finishes(attach.Thread, "the failed attach"); + + Assert.That(attach.Error(), Is.Not.Null); + Assert.That(store.IsAttachedToACache, Is.False); + Assert.That(store.Attached, Is.Null, "the registry was taken back"); + } + + [Test] + public void AttachingNoStoreIsRefusedBeforeAnythingIsHeld() + { + using var cache = new CultCache(CultDocumentRegistry.ForTypes(new[] { typeof(IdDeck) })); + + Assert.Throws(() => cache.AddBackingStore(null!)); + } + + private const string ReentryRefusal = "A cacheless write was made inside a call that already holds this store"; + + private const string AttachedRefusal = "The store is attached to a cache, write through the cache."; + + private const string JournalRefusal = "A journal entered the cache it journals"; + + // Every store entry point, called by a journal the attach's pull runs on the attaching thread, through a real attach. The store is + // attached for its whole attach, so each call is refused as it is after the attach, and nothing reaches the file or the store's + // staging that the cache never admitted: after a flush the attaching cache and a fresh cache over the file hold the same keys. + [TestCase("single", "pullall")] + [TestCase("single", "push")] + [TestCase("single", "delete")] + [TestCase("single", "pushall")] + [TestCase("single", "push+pushall")] + [TestCase("single", "delete+pushall")] + [TestCase("single", "commit")] + [TestCase("single", "judge")] + [TestCase("single", "stage")] + [TestCase("single", "snapshot")] + [TestCase("single", "writedoc")] + [TestCase("single", "writerecord")] + [TestCase("directory", "pullall")] + [TestCase("directory", "push")] + [TestCase("directory", "delete")] + [TestCase("directory", "pushall")] + [TestCase("directory", "push+pushall")] + [TestCase("directory", "delete+pushall")] + [TestCase("directory", "commit")] + [TestCase("directory", "judge")] + [TestCase("directory", "stage")] + public async Task AJournalOnTheAttachingThreadCannotReachItsStore(string kind, string op) + { + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(IdDeck) }); + var descriptor = registry.GetRequired(typeof(IdDeck)); + var path = kind == "single" ? _path : Path.Combine(_directory, "dir"); + CacheBackingStore Open() => kind == "single" + ? new SingleFileMessagePackBackingStore(path) + : new DirectoryMessagePackBackingStore(path); + + using (var seed = new CultCache(registry, Codec(registry))) + { + seed.AddBackingStore(Open()); + await seed.UpsertAsync(typeof(IdDeck), new IdDeck { Name = "a" }, new CultRecordKey("a")); + seed.FlushAllBackingStores(); + } + CultPersistedRecord? recordA = null; + CultSchemaCatalogEntry? entryA = null; + if (kind == "single") + { + var snapshot = new SingleFileMessagePackBackingStore(path).ReadStoreSnapshot()!; + recordA = snapshot.Records.First(); + entryA = snapshot.SchemaCatalog.First(); + } + + var store = Open(); + using var cache = new CultCache(registry, Codec(registry)); + var docB = new CultStoredDocument(new CultRecordKey("b"), DateTimeOffset.UtcNow.ToString("O"), descriptor, new IdDeck { Name = "b" }); + var journalRuns = 0; + Exception? journalError = null; + using var journal = cache.AddJournal(_ => + { + if (journalRuns++ > 0) + return; + try + { + switch (op) + { + case "pullall": store.PullAll(); break; + case "push": store.Push(docB); break; + case "delete": store.Delete(cache.GetStored(new CultRecordKey("a"))!); break; + case "pushall": store.PushAll(); break; + case "push+pushall": store.Push(docB); store.PushAll(); break; + case "delete+pushall": store.Delete(cache.GetStored(new CultRecordKey("a"))!); store.PushAll(); break; + case "commit": + store.CommitBatch(new CultCommitRequest(new[] { docB }, Array.Empty(), Array.Empty(), + Array.Empty<(CultRecordKey, string?)>(), false), wait: true); + break; + case "judge": store.JudgeChecks(new[] { "a" }, wait: true); break; + case "stage": store.RefuseRemovalOfUndeclared(new[] { "zz" }); break; + case "snapshot": ((SingleFileBackingStore)store).ReadStoreSnapshot(); break; + case "writedoc": + ((SingleFileBackingStore)store).WriteDocument(new CultRecordKey("b"), new IdDeck { Name = "b" }, registry: registry); + break; + case "writerecord": ((SingleFileBackingStore)store).WriteRecord(recordA!, entryA!, registry: registry); break; + default: throw new ArgumentException(op); + } + } + catch (Exception error) { journalError = error; } + }); + + cache.AddBackingStore(store); + + Assert.That(journalRuns, Is.GreaterThan(0), "the pull's admission ran the journal"); + Assert.That(journalError, Is.InstanceOf()); + Assert.That(journalError!.Message, Does.StartWith(op is "writedoc" or "writerecord" ? AttachedRefusal : JournalRefusal)); + Assert.That(store.IsDirty, Is.False, "nothing the cache never admitted is staged"); + + await cache.UpsertAsync(typeof(IdDeck), new IdDeck { Name = "later" }, new CultRecordKey("later")); + Assert.DoesNotThrow(() => cache.FlushAllBackingStores()); + Assert.DoesNotThrow(() => store.PullAll(), "the refused call did not take back the attach's registry"); + using var fresh = new CultCache(registry, Codec(registry)); + fresh.AddBackingStore(Open()); + string[] Keys(CultCache from) => from.AllStoredDocuments.Select(document => document.Key.Value).OrderBy(key => key, StringComparer.Ordinal).ToArray(); + Assert.That(Keys(cache), Does.Contain("a").And.Contain("later").And.Not.Contain("b")); + Assert.That(Keys(cache), Is.EqualTo(Keys(fresh)), "the attaching cache and a fresh cache over the same file agree"); + } + + // A cacheless write made inside another store's hold is refused as entering a second store, not as re-entering its own: the + // second-store rule is checked before the same-store one, so the message sends the caller to the right place. + [Test] + public void ACachelessWriteInsideAnotherStoresHoldGetsTheSecondStoreRefusal() + { + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(TapDoc) }); + var pathB = Path.Combine(_directory, "second.cc"); + var storeA = new RoleStore(_path); + var storeB = new RoleStore(pathB); + TapDoc.Hook = () => storeB.WriteDocument(new CultRecordKey("inner"), new TapDoc { Name = "inner" }, registry: registry); + + var outer = Assert.Catch(() => storeA.WriteDocument(new CultRecordKey("outer"), new TapDoc { Name = "outer" }, registry: registry)); + + var refusal = outer!; + while (refusal is not InvalidOperationException && refusal.InnerException != null) + refusal = refusal.InnerException; + while (refusal.InnerException is InvalidOperationException deeper) + refusal = deeper; + Assert.That(refusal, Is.InstanceOf()); + Assert.That(refusal.Message, Does.StartWith("A store's gate was entered inside another store's hold")); + Assert.That(File.Exists(_path), Is.False); + Assert.That(File.Exists(pathB), Is.False); + } + + // A cacheless write owns its store outright: user code another cacheless write of the same store runs may not make a second. + [Test] + public void ACachelessWriteInsideACachelessWriteOfTheSameStoreIsRefused() + { + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(TapDoc) }); + var store = new RoleStore(_path); + TapDoc.Hook = () => store.WriteDocument(new CultRecordKey("inner"), new TapDoc { Name = "inner" }, registry: registry); + + var outer = Assert.Catch(() => store.WriteDocument(new CultRecordKey("outer"), new TapDoc { Name = "outer" }, registry: registry)); + + Assert.That(Refused(outer), Is.True, "the outer write failed with the inner call's refusal"); + var refusal = outer!; + while (refusal is not InvalidOperationException && refusal.InnerException != null) + refusal = refusal.InnerException; + Assert.That(refusal.Message, Does.StartWith(ReentryRefusal)); + Assert.That(File.Exists(_path), Is.False); + Assert.That(store.Attached, Is.Null); + } + + // The bound on a call's gates is structural: an attach publishes Cache and takes it back inside one hold of the store's gate, so a + // cache read under that gate is settled. This test publishes and retracts in two holds, the only way to miss; the reader, which + // settled on the cache and then finds it gone under that cache's gate, throws. + [Test] + public void ACallWhoseSettledCacheChangesThrows() + { + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(IdDeck) }); + new SingleFileMessagePackBackingStore(_path).WriteDocument(new CultRecordKey("a"), new IdDeck { Name = "a" }, registry: registry); + var store = new RoleStore(_path); + using var cache = new CultCache(registry); + var holding = new ManualResetEventSlim(); + var publish = new ManualResetEventSlim(); + var retract = new ManualResetEventSlim(); + var tamper = OnThread("tamper", () => cache.Held(() => + { + store.UnderStoreGate(() => + { + holding.Set(); + publish.Wait(TimeSpan.FromSeconds(30)); + store.Cache = cache; + return true; + }); + retract.Wait(TimeSpan.FromSeconds(30)); + store.UnderStoreGate(() => { store.Cache = null; return true; }); + return true; + })); + Assert.That(holding.Wait(TimeSpan.FromSeconds(10)), Is.True); + Exception? readerError = null; + var reader = OnThread("reader", () => + { + try { store.ReadStoreSnapshot(); } + catch (Exception e) { readerError = e; throw; } + }); + WaitUntilBlocked(reader.Thread); + publish.Set(); + Thread.Sleep(500); + retract.Set(); + Finishes(tamper.Thread, "the tampering thread"); + Finishes(reader.Thread, "the reader"); + + Assert.That(tamper.Error(), Is.Null); + Assert.That(readerError, Is.InstanceOf().With.Message.StartWith("A store's settled cache changed")); + } + + // Failing attaches over and over, from one, two and four caches, against readers: a reader's gate changes under it at most once + // (the bound above throws on a second), so no reader fails and every thread finishes. + [TestCase(1)] + [TestCase(2)] + [TestCase(4)] + public void ReadersAcrossFailingAttachChurnNeverExceedTheBound(int attachers) + { + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(IdDeck) }); + new SingleFileMessagePackBackingStore(_path).WriteDocument(new CultRecordKey("a"), new IdDeck { Name = "a" }, registry: registry); + var store = new RoleStore(_path) { FailAttachPull = true }; + store.ReleaseAttach.Set(); + var caches = Enumerable.Range(0, attachers).Select(_ => new CultCache(registry)).ToArray(); + var stop = 0; + var readerErrors = new System.Collections.Concurrent.ConcurrentBag(); + var attachErrors = new System.Collections.Concurrent.ConcurrentBag(); + var threads = caches.Select(c => OnThread("attach", () => + { + while (Volatile.Read(ref stop) == 0) + { + try { c.AddBackingStore(store); } + catch (InvalidOperationException e) when (e.Message.Contains("pull fails")) { } + catch (Exception e) { attachErrors.Add(e.Message); } + } + })).Concat(Enumerable.Range(0, 4).Select(_ => OnThread("churn-reader", () => + { + while (Volatile.Read(ref stop) == 0) + { + try { store.ReadStoreSnapshot(); } + catch (Exception e) { readerErrors.Add(e.Message); } + } + }))).ToArray(); + Thread.Sleep(TimeSpan.FromSeconds(2)); + Volatile.Write(ref stop, 1); + foreach (var t in threads) + Finishes(t.Thread, "a churn thread"); + + Assert.That(readerErrors, Is.Empty); + Assert.That(attachErrors, Is.Empty); + foreach (var c in caches) + c.Dispose(); + } + + // A call on a detached store takes exactly one gate (the store's), and one on a store whose attach has settled takes exactly + // one (the settled cache's, reached by the guess): the counter counts each take. + [Test] + public void OneStoreCallTakesExactlyOneGateOnADetachedStoreAndOnASettledOne() + { + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(IdDeck) }); + new SingleFileMessagePackBackingStore(_path).WriteDocument(new CultRecordKey("a"), new IdDeck { Name = "a" }, registry: registry); + var store = new RoleStore(_path); + store.ReleaseAttach.Set(); + + CacheBackingStore.GateTakes = 0; + store.ReadStoreSnapshot(); + Assert.That(CacheBackingStore.GateTakes, Is.EqualTo(1), "a detached store's call takes the store's gate"); + + using var cache = new CultCache(registry); + cache.AddBackingStore(store); + CacheBackingStore.GateTakes = 0; + store.ReadStoreSnapshot(); + Assert.That(CacheBackingStore.GateTakes, Is.EqualTo(1), "a settled store's call takes only its cache's gate"); + } + + // Three gates are reached only by a stale guess that a later attach outdates: the call read the cache a failing attach had + // published (its first gate, taken only to find Cache moved), then the store's gate (its second) found another cache settled + // by an attach that won the store in between (its third). One round is ordered with the store's hooks, and no call exceeds three. + [Test] + public void OneStoreCallTakesThreeGatesWhenAStaleGuessIsFollowedBySuccessAndNeverMore() + { + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(IdDeck) }); + new SingleFileMessagePackBackingStore(_path).WriteDocument(new CultRecordKey("a"), new IdDeck { Name = "a" }, registry: registry); + var mostTakes = 0; + for (var round = 0; round < 20 && mostTakes < 3; round++) + { + var store = new RoleStore(_path) { FailAttachPull = true }; + using var failing = new CultCache(registry); + using var settling = new CultCache(registry); + var failed = OnThread("attach", () => failing.AddBackingStore(store)); + Assert.That(store.AttachInside.Wait(TimeSpan.FromSeconds(10)), Is.True, "the failing attach is inside its pull, Cache published"); + var settled = OnThread("settling", () => settling.AddBackingStore(store)); + WaitUntilBlocked(settled.Thread); + var takes = 0; + var call = OnThread("call", () => + { + CacheBackingStore.GateTakes = 0; + store.ReadStoreSnapshot(); + takes = CacheBackingStore.GateTakes; + }); + WaitUntilBlocked(call.Thread); + Thread.Sleep(100); + store.ReleaseAttach.Set(); + Finishes(failed.Thread, "the failing attach"); + Finishes(settled.Thread, "the settling attach"); + Finishes(call.Thread, "the call"); + + Assert.That(Refused(failed.Error()), Is.True, "the first attach failed"); + Assert.That(settled.Error(), Is.Null, "the second attach settled"); + Assert.That(takes, Is.InRange(1, 3), "no call took more than three gates"); + mostTakes = Math.Max(mostTakes, takes); + } + + Assert.That(mostTakes, Is.EqualTo(3), "a call that read a stale guess and then met a settled cache took three gates"); + } + + // Under attaches that fail, Cache read under the store's gate is always null (an attach publishes and retracts it inside one + // hold of that gate), so one call takes at most two gates: the guessed cache's, then the store's. A retry around the unlocked + // guess would take one per stale guess and exceed this. + [Test] + public void OneStoreCallTakesAtMostTwoGatesUnderFailingAttachChurn() + { + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(IdDeck) }); + new SingleFileMessagePackBackingStore(_path).WriteDocument(new CultRecordKey("a"), new IdDeck { Name = "a" }, registry: registry); + var store = new RoleStore(_path) { FailAttachPull = true }; + store.ReleaseAttach.Set(); + var caches = Enumerable.Range(0, 4).Select(_ => new CultCache(registry)).ToArray(); + var stop = 0; + var mostTakes = 0; + var calls = 0; + var threads = caches.Select(c => OnThread("attach", () => + { + while (Volatile.Read(ref stop) == 0) + { + try { c.AddBackingStore(store); } + catch (InvalidOperationException e) when (e.Message.Contains("pull fails")) { } + } + })).Concat(Enumerable.Range(0, 4).Select(_ => OnThread("counting-reader", () => + { + while (Volatile.Read(ref stop) == 0) + { + CacheBackingStore.GateTakes = 0; + try { store.ReadStoreSnapshot(); } + catch (InvalidOperationException) { } + var takes = CacheBackingStore.GateTakes; + Interlocked.Increment(ref calls); + int seen; + while (takes > (seen = Volatile.Read(ref mostTakes)) && Interlocked.CompareExchange(ref mostTakes, takes, seen) != seen) { } + } + }))).ToArray(); + Thread.Sleep(TimeSpan.FromSeconds(3)); + Volatile.Write(ref stop, 1); + foreach (var t in threads) + Finishes(t.Thread, "a churn thread"); + + Assert.That(calls, Is.GreaterThan(100), "the readers made calls under churn"); + Assert.That(mostTakes, Is.LessThanOrEqualTo(2), "no call took more than two gates"); + foreach (var c in caches) + c.Dispose(); + } + } +} diff --git a/tests/GameCult.Caching.Tests/CompatibleSchemaIdTests.cs b/tests/GameCult.Caching.Tests/CompatibleSchemaIdTests.cs new file mode 100644 index 00000000..3ef2816d --- /dev/null +++ b/tests/GameCult.Caching.Tests/CompatibleSchemaIdTests.cs @@ -0,0 +1,579 @@ +#nullable enable +using System; +using System.IO; +using System.Linq; +using GameCult.Caching.MessagePack; +using MessagePack; +using NUnit.Framework; +using static GameCult.Caching.Tests.EmittedDocumentTypes; + +namespace GameCult.Caching.Tests +{ + // A document type declares the older schema ids its records may sit under (CultDocumentAttribute.CompatibleSchemaIds). The + // registered catalog entry lists them, a cache resolves a record under one to that type and stamps it with the type's own id, + // and a store writer never changes the identity of a record it did not receive. + public class CompatibleSchemaIdTests + { + private const string OldId = "old.id"; + private const string OwnerId = "sha256:86186abf7d63d8a326c725d39a10f3a4db11b1e8698e8c0fcf6d9720031bc480"; + + [CultDocument("tests.compat_deck", "tests.compat_deck.v2", CompatibleSchemaIds = new[] { OldId, OldId })] + [MessagePackObject] + public sealed class DeclaringDeck + { + [Key(0)] [CultName] public string Name { get; set; } = ""; + } + + // The same schema as DeclaringDeck (the class name is not part of a schema id), declaring nothing. Emitted, because a + // declared type that differs from DeclaringDeck only in its declaration is refused beside it by the shared registry. + private static readonly Type BareDeck = Emit("BareDeck", "tests.compat_deck", "tests.compat_deck.v2", new[] { new Field("Name", typeof(string), 0, IsName: true) }); + + [CultDocument("tests.compat_owner", "tests.compat_owner.v1", CompatibleSchemaIds = new[] { OwnerId, " " })] + [MessagePackObject] + public sealed class Owner + { + [Key(0)] [CultName] public string Name { get; set; } = ""; + } + + // One schema before and after a member was added, with nothing declared: the later type reads the earlier records by name. + [CultDocument("tests.compat_evolving", "tests.compat_evolving.v1")] + [MessagePackObject] + public sealed class EvolvingV1 + { + [Key(0)] [CultName] public string Name { get; set; } = ""; + } + + [CultDocument("tests.compat_evolving", "tests.compat_evolving.v2")] + [MessagePackObject] + public sealed class EvolvingV2 + { + [Key(0)] [CultName] public string Name { get; set; } = ""; + [Key(1)] public string Extra { get; set; } = ""; + } + + private static readonly CultDocumentRegistry Declaring = CultDocumentRegistry.ForTypes(new[] { typeof(DeclaringDeck) }); + private static readonly CultDocumentRegistry Bare = CultDocumentRegistry.ForTypes(new[] { BareDeck }); + private static readonly CultRecordKey D = new("d"); + private string _directory = ""; + + [SetUp] + public void SetUp() + { + _directory = Path.Combine(Path.GetTempPath(), "cultlib-compat-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(_directory); + } + + [TearDown] + public void TearDown() + { + if (Directory.Exists(_directory)) + Directory.Delete(_directory, true); + } + + private static CultCache Open(string path, CultDocumentRegistry registry) => + CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = registry }); + + private static CultPersistedStoreSnapshot Read(string path) => + CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + + private static object Deck(string name) => New(BareDeck, ("Name", name)); + + private static void Rewrite(string path, Action edit) + { + var snapshot = Read(path); + edit(snapshot); + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + } + + // A store written by a runtime that knew the schema under an older id and another name: one entry that owns OldId, one + // record under it. Nothing but the declaration lets this runtime's type claim it. + private string OldStore(string name) + { + var path = Path.Combine(_directory, name); + using (var cache = Open(path, Bare)) + cache.Commit(batch => batch.Upsert(BareDeck, Deck("d"), D)); + Rewrite(path, snapshot => + { + var entry = snapshot.SchemaCatalog.Single(); + entry.SchemaId = OldId; + entry.SchemaName = "tests.legacy_deck"; + entry.ContentHash = "stale"; + entry.CompatibleSchemaIds = new[] { OldId }; + snapshot.Records.Single().SchemaId = OldId; + }); + return path; + } + + // The same store with the entry keeping this schema's name and only listing the older id: what a store carries after a + // writer that did not declare it, so a runtime without the declaration can still read it. + private string OldStoreUnderTheSameName(string name) + { + var path = Path.Combine(_directory, name); + using (var cache = Open(path, Bare)) + cache.Commit(batch => batch.Upsert(BareDeck, Deck("d"), D)); + Rewrite(path, snapshot => + { + var entry = snapshot.SchemaCatalog.Single(); + entry.ContentHash = "stale"; + entry.CompatibleSchemaIds = new[] { entry.SchemaId, OldId }; + snapshot.Records.Single().SchemaId = OldId; + }); + return path; + } + + [Test] + public void TheEmittedBareDeckIsTheDeclaringDecksSchema() + { + Assert.That(Bare.GetRequired(BareDeck).SchemaId, Is.EqualTo(Declaring.GetRequired().SchemaId)); + } + + [Test] + public void ARecordUnderADeclaredOldIdOpensAndACommitRewritesItUnderTheRegisteredId() + { + var path = OldStore("old.cc"); + var descriptor = Declaring.GetRequired(); + + using (var cache = Open(path, Declaring)) + { + Assert.That(cache.Get(D)!.Name, Is.EqualTo("d"), "the declaration resolves the record to the type"); + Assert.That(cache.Commit(batch => batch.Upsert(typeof(DeclaringDeck), new DeclaringDeck { Name = "d2" }, D)), Is.True); + } + + var written = Read(path); + Assert.That(written.Records.Single().SchemaId, Is.EqualTo(descriptor.SchemaId)); + var entry = written.SchemaCatalog.Single(); + Assert.That(entry.SchemaId, Is.EqualTo(descriptor.SchemaId)); + Assert.That(entry.SchemaName, Is.EqualTo("tests.compat_deck")); + Assert.That(entry.CompatibleSchemaIds, Is.EqualTo(new[] { descriptor.SchemaId, OldId })); + foreach (var registry in new[] { Declaring, Bare }) + { + using var reopened = Open(path, registry); + Assert.That(reopened.AllEntries.Count(), Is.EqualTo(1)); + } + } + + // The name a catalog entry carries is metadata: a record resolves by its schema id, so a schema renamed under the id the + // registered type owns opens, with the entry's content hash no longer matching the type's. + [Test] + public void ASchemaRenamedUnderTheIdARegisteredTypeOwnsOpens() + { + var path = Path.Combine(_directory, "renamed.cc"); + using (var cache = Open(path, Bare)) + cache.Commit(batch => batch.Upsert(BareDeck, Deck("d"), D)); + Rewrite(path, snapshot => + { + snapshot.SchemaCatalog.Single().SchemaName = "tests.renamed_deck"; + snapshot.SchemaCatalog.Single().ContentHash = "stale"; + }); + + using var reopened = Open(path, Bare); + Assert.That(EmittedDocumentTypes.Read(reopened.Get(D)!, "Name"), Is.EqualTo("d")); + } + + [Test] + public void ARecordUnderAnOldIdNoTypeDeclaresIsCarriedAsForeignNotOpenedByThatSchemaName() + { + var path = OldStore("undeclared.cc"); + + using var cache = Open(path, Bare); + Assert.That(cache.Get(D), Is.Null, "the entry's name is metadata; only a declared id resolves it"); + Assert.That(cache.BackingStores.Single().ForeignRecords.Select(record => (record.Key, record.SchemaId, record.SchemaName)), + Is.EqualTo(new[] { (D.Value, OldId, "tests.legacy_deck") })); + } + + // An unconditional commit writes what it staged: the record the batch does not name is copied as the file holds it, under + // the old id and with the entry that publishes that id, though the cache resolved it to the registered type. + [Test] + public void AnUnconditionalCommitCopiesALoadedRecordUnderItsOldIdAsTheFileHoldsIt() + { + var path = OldStore("copied.cc"); + var descriptor = Declaring.GetRequired(); + var before = Read(path); + var entryBefore = StoreSlices.CatalogEntry(path, OldId); + + using (var cache = Open(path, Declaring)) + cache.Commit(batch => batch.Upsert(typeof(DeclaringDeck), new DeclaringDeck { Name = "e" }, new CultRecordKey("e"))); + + var written = Read(path); + Assert.That(written.Records.Single(record => record.Key == "d").SchemaId, Is.EqualTo(OldId)); + Assert.That(written.Records.Single(record => record.Key == "e").SchemaId, Is.EqualTo(descriptor.SchemaId)); + var copied = written.Records.Single(record => record.Key == "d"); + var original = before.Records.Single(); + Assert.That((copied.StoredAt, copied.Payload), Is.EqualTo((original.StoredAt, original.Payload))); + Assert.That(StoreSlices.CatalogEntry(path, OldId), Is.EqualTo(entryBefore), + "the entry that published the old id is carried as its bytes"); + } + + // A commit onto the file writes its batch and leaves every other record as the file holds it: the store writer never + // changes identity. The record stays under the older id, which the registered entry publishes as compatible. + [Test] + public void ACommitOntoTheFileLeavesARecordUnderADeclaredOldIdAsItIsAndPublishesTheId() + { + var path = OldStoreUnderTheSameName("onto.cc"); + var descriptor = Declaring.GetRequired(); + + using (var cache = Open(path, Declaring)) + { + Assert.That(cache.Get(D), Is.Not.Null); + Assert.That(cache.Commit(batch => + { + batch.Expect(new CultRecordKey("e"), null); + batch.Upsert(typeof(DeclaringDeck), new DeclaringDeck { Name = "e" }, new CultRecordKey("e")); + }), Is.True); + } + + var written = Read(path); + Assert.That(written.Records.Single(record => record.Key == "d").SchemaId, Is.EqualTo(OldId)); + Assert.That(written.Records.Single(record => record.Key == "e").SchemaId, Is.EqualTo(descriptor.SchemaId)); + Assert.That(written.SchemaCatalog.Single().CompatibleSchemaIds, Does.Contain(OldId)); + using var reopened = Open(path, Declaring); + Assert.That(reopened.Get(D), Is.Not.Null); + } + + [Test] + public void WithoutTheDeclarationACommitOntoTheFileIsRefusedTypedAndTheFileLeftAsItWas() + { + var path = OldStoreUnderTheSameName("undeclared-onto.cc"); + var bytes = File.ReadAllBytes(path); + + using var cache = Open(path, Bare); + var refusal = Assert.Throws(() => cache.Commit(batch => + { + batch.Expect(new CultRecordKey("e"), null); + batch.Upsert(BareDeck, Deck("e"), new CultRecordKey("e")); + }))!; + Assert.That(refusal.SchemaId, Is.EqualTo(OldId)); + Assert.That(File.ReadAllBytes(path), Is.EqualTo(bytes)); + } + + // A record under a declared old id is the same record the cache holds, loaded under that id: a condition naming it holds. + [Test] + public void AConditionOnARecordUnderADeclaredOldIdHoldsAndTheWholeStoreCountsAsUnchanged() + { + var path = OldStoreUnderTheSameName("conditions.cc"); + + using var cache = Open(path, Declaring); + var current = cache.Get(D)!; + Assert.That(cache.TryCommit(batch => + { + batch.Expect(D, current); + batch.ExpectUnchanged(); + batch.Upsert(typeof(DeclaringDeck), new DeclaringDeck { Name = "e" }, new CultRecordKey("e")); + }), Is.EqualTo(CultCommitOutcome.Committed)); + } + + // The directory store judges conditions against its manifest, whose record still sits under the older id. + [Test] + public void AConditionOnARecordUnderADeclaredOldIdHoldsInTheDirectoryStore() + { + var path = Path.Combine(_directory, "dir.cc"); + var options = new CultCacheOpenOptions { Registry = Bare, UseDirectoryStore = true }; + using (var cache = CultCacheMessagePack.Create(path, options)) + cache.Commit(batch => batch.Upsert(BareDeck, Deck("d"), D)); + Rewrite(path, manifest => + { + var entry = manifest.SchemaCatalog.Single(); + entry.ContentHash = "stale"; + entry.CompatibleSchemaIds = new[] { entry.SchemaId, OldId }; + manifest.Records.Single().SchemaId = OldId; + }); + + using var declaring = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Declaring, UseDirectoryStore = true }); + var current = declaring.Get(D)!; + Assert.That(declaring.TryCommit(batch => + { + batch.Expect(D, current); + batch.ExpectUnchanged(); + batch.Upsert(typeof(DeclaringDeck), new DeclaringDeck { Name = "e" }, new CultRecordKey("e")); + }), Is.EqualTo(CultCommitOutcome.Committed)); + } + + // Conditions compare the id a record was loaded under with the file's, exactly. A record resolved by some other path than + // its type's own id is still unchanged when nothing on disk moved: by its schema name under a foreign id, by name across a + // schema change that declared nothing, or by a file entry that lists this runtime's id. + private static void AssertUnchangedConditionsHold(CultCache cache, CultRecordKey key, object current, Type type, object added) + { + Assert.That(cache.TryCommit(batch => + { + batch.Expect(key, current); + batch.Upsert(type, added, new CultRecordKey("added-" + Guid.NewGuid().ToString("N"))); + }), Is.EqualTo(CultCommitOutcome.Committed), "Expect"); + Assert.That(cache.TryCommit(batch => + { + batch.ExpectUnchanged(); + batch.Upsert(type, added, new CultRecordKey("added-" + Guid.NewGuid().ToString("N"))); + }), Is.EqualTo(CultCommitOutcome.Committed), "ExpectUnchanged"); + } + + [Test] + public void AConditionOnARecordResolvedByNameUnderAForeignIdHolds() + { + var path = Path.Combine(_directory, "foreign.cc"); + using (var cache = Open(path, Bare)) + cache.Commit(batch => batch.Upsert(BareDeck, Deck("d"), D)); + Rewrite(path, snapshot => + { + var entry = snapshot.SchemaCatalog.Single(); + entry.SchemaId = "foreign.id"; + entry.CompatibleSchemaIds = new[] { "foreign.id" }; + snapshot.Records.Single().SchemaId = "foreign.id"; + }); + + using var reader = Open(path, Declaring); + AssertUnchangedConditionsHold(reader, D, reader.Get(D)!, typeof(DeclaringDeck), new DeclaringDeck { Name = "e" }); + } + + [Test] + public void AConditionOnARecordAnEvolvedTypeResolvedByNameHolds() + { + var path = Path.Combine(_directory, "evolved.cc"); + using (var cache = Open(path, CultDocumentRegistry.ForTypes(new[] { typeof(EvolvingV1) }))) + cache.Commit(batch => batch.Upsert(typeof(EvolvingV1), new EvolvingV1 { Name = "d" }, D)); + + using var reader = Open(path, CultDocumentRegistry.ForTypes(new[] { typeof(EvolvingV2) })); + AssertUnchangedConditionsHold(reader, D, reader.Get(D)!, typeof(EvolvingV2), new EvolvingV2 { Name = "e" }); + } + + [Test] + public void AConditionOnARecordAFileEntryListingTheLocalIdResolvedHolds() + { + var path = Path.Combine(_directory, "listed.cc"); + var localId = Bare.GetRequired(BareDeck).SchemaId; + using (var cache = Open(path, Bare)) + cache.Commit(batch => batch.Upsert(BareDeck, Deck("d"), D)); + Rewrite(path, snapshot => + { + var entry = snapshot.SchemaCatalog.Single(); + entry.SchemaId = OldId; + entry.CompatibleSchemaIds = new[] { OldId, localId }; + snapshot.Records.Single().SchemaId = OldId; + }); + + using var reader = Open(path, Bare); + AssertUnchangedConditionsHold(reader, D, reader.Get(D)!, BareDeck, Deck("e")); + } + + // A plain record and a variant loaded under a declared old id carry that id, so a condition on either holds while the file + // is unchanged. An unconditional commit then copies both as they are, still under the old id, and the cache still holds + // each under it: the same conditions hold against the rewritten file. + [Test] + public void ConditionsOnRecordsLoadedUnderAnOldIdHoldBeforeAndAfterAnUnconditionalCommit() + { + var path = Path.Combine(_directory, "restamped.cc"); + var b = new CultRecordKey("b"); + var v = new CultRecordKey("v"); + using (var cache = Open(path, Declaring)) + { + cache.Commit(batch => batch.Upsert(typeof(DeclaringDeck), new DeclaringDeck { Name = "b" }, b)); + cache.Commit(batch => batch.UpsertVariant(v, b, new[] { cache.Override(typeof(DeclaringDeck), nameof(DeclaringDeck.Name), "v") })); + } + + Rewrite(path, snapshot => + { + foreach (var record in snapshot.Records) + record.SchemaId = OldId; + }); + + using var reader = Open(path, Declaring); + void ConditionsHold(string phase) + { + foreach (var key in new[] { b, v }) + { + Assert.That(reader.TryCommit(batch => + { + batch.Expect(key, reader.Get(key)!); + batch.Upsert(typeof(DeclaringDeck), new DeclaringDeck { Name = "f" }, new CultRecordKey($"f-{phase}-{key.Value}")); + }), Is.EqualTo(CultCommitOutcome.Committed), $"{phase}: Expect {key.Value}"); + } + + Assert.That(reader.TryCommit(batch => + { + batch.ExpectUnchanged(); + batch.Upsert(typeof(DeclaringDeck), new DeclaringDeck { Name = "g" }, new CultRecordKey($"g-{phase}")); + }), Is.EqualTo(CultCommitOutcome.Committed), $"{phase}: ExpectUnchanged"); + } + + ConditionsHold("loaded"); + Assert.That(Read(path).Records.Where(record => record.Key is "b" or "v").Select(record => record.SchemaId), Is.All.EqualTo(OldId), + "commits onto the file left the loaded records under the old id"); + + reader.Commit(batch => batch.Upsert(typeof(DeclaringDeck), new DeclaringDeck { Name = "e" }, new CultRecordKey("e"))); + Assert.That(Read(path).Records.Where(record => record.Key is "b" or "v").Select(record => record.SchemaId), Is.All.EqualTo(OldId), + "the unconditional commit copied the loaded records under the old id"); + ConditionsHold("rewritten"); + } + + // The directory store rereads a record whose manifest entry moved to another id with the same storedAt, as a rewrite that + // mints no storedAt leaves it: after a pull the cache holds the record under that id, and a condition on it holds. + [Test] + public void ADirectoryStorePullRereadsARecordWhoseIdAloneChanged() + { + var path = Path.Combine(_directory, "dir-moved.cc"); + var options = new CultCacheOpenOptions { Registry = Declaring, UseDirectoryStore = true }; + using (var writer = CultCacheMessagePack.Create(path, options)) + writer.Commit(batch => batch.Upsert(typeof(DeclaringDeck), new DeclaringDeck { Name = "d" }, D)); + + using var cache = CultCacheMessagePack.Create(path, options); + Rewrite(path, manifest => manifest.Records.Single().SchemaId = OldId); + cache.PullAllBackingStoresAsync().GetAwaiter().GetResult(); + + Assert.That(cache.TryCommit(batch => + { + batch.Expect(D, cache.Get(D)!); + batch.Upsert(typeof(DeclaringDeck), new DeclaringDeck { Name = "e" }, new CultRecordKey("e")); + }), Is.EqualTo(CultCommitOutcome.Committed)); + } + + // A rolling deployment shares one single-file store between v1 (one member) and v2 (a second member, declaring v1's id). + private static readonly Field RollName = new("Name", typeof(string), 0, IsName: true); + private static readonly Type RollV1 = Emit("RollV1", "tests.compat_roll", "tests.compat_roll.v1", new[] { RollName }); + private static readonly string RollV1Id = CultDocumentRegistry.ForTypes(new[] { RollV1 }).GetRequired(RollV1).SchemaId; + private static readonly Type RollV2 = Emit("RollV2", "tests.compat_roll", "tests.compat_roll.v2", new[] { RollName, new Field("Extra", typeof(string), 1) }, new[] { RollV1Id }); + private static readonly CultRecordKey RollK = new("k"); + private static readonly CultRecordKey RollY = new("y"); + + private CultCache OpenRoll(Type type) => Open(Path.Combine(_directory, "roll.cc"), CultDocumentRegistry.ForTypes(new[] { type })); + private string RollPath => Path.Combine(_directory, "roll.cc"); + private static string Extra(object? document) => (string?)EmittedDocumentTypes.Read(document!, "Extra") ?? ""; + + // Writes k with Extra "important" as v2. + private void WriteImportant() + { + using var writer = OpenRoll(RollV2); + writer.Commit(batch => batch.Upsert(RollV2, New(RollV2, ("Name", "k"), ("Extra", "important")), RollK)); + } + + // A write by a cache of the given version, through a commit or a plain flush, that leaves the key set as it was. + private void UnrelatedWrite(Type version, bool viaFlush) + { + using var cache = OpenRoll(version); + Assert.That(EmittedDocumentTypes.Read(cache.Get(RollK)!, "Name"), Is.EqualTo("k"), $"{version.Name} reads k"); + if (viaFlush) + { + cache.BackingStores.Single().PushAll(); + return; + } + + cache.Commit(batch => batch.Upsert(version, New(version, ("Name", "z")), new CultRecordKey("z"))); + cache.Commit(batch => batch.Remove(new CultRecordKey("z"))); + } + + // The derivation a v2 cache commits from the k it holds, under a condition on that k. + private static CultCommitOutcome Derive(CultCache cache, object held, bool unchangedOnly) => cache.TryCommit(batch => + { + if (unchangedOnly) + batch.ExpectUnchanged(); + else + batch.Expect(RollK, held); + batch.Upsert(RollV2, New(RollV2, ("Name", "y"), ("Extra", "derived-from-" + Extra(held))), RollY); + }); + + // v1 reads v2's record through the entry that lists v1's id: an id v1 does not own. A write of another record by v1 copies k + // exactly as v2 stored it, the member v1 lacks included, under v2's id and storedAt, so a v2 cache's condition on k still holds + // and what it derives from that member lands. + [Test] + public void AnUnrelatedWriteByAnOlderVersionLeavesANewerRecordAsStored([Values] bool unchangedOnly, [Values] bool viaFlush) + { + WriteImportant(); + var written = Read(RollPath); + + using var newer = OpenRoll(RollV2); + var current = newer.Get(RollK)!; + UnrelatedWrite(RollV1, viaFlush); + + var rewritten = Read(RollPath); + var record = rewritten.Records.Single(); + Assert.That((record.SchemaId, record.StoredAt, record.Payload), Is.EqualTo((written.Records.Single().SchemaId, written.Records.Single().StoredAt, written.Records.Single().Payload))); + Assert.That(CultDocumentMessagePackSerialization.SerializeSnapshot(new CultPersistedStoreSnapshot { SchemaCatalog = rewritten.SchemaCatalog }), + Is.EqualTo(CultDocumentMessagePackSerialization.SerializeSnapshot(new CultPersistedStoreSnapshot { SchemaCatalog = written.SchemaCatalog })), + "v2's catalog entry is written as v2 wrote it"); + + Assert.That(Derive(newer, current, unchangedOnly), Is.EqualTo(CultCommitOutcome.Committed)); + using var reopened = OpenRoll(RollV2); + Assert.That(Extra(reopened.Get(RollK)), Is.EqualTo("important")); + Assert.That(Extra(reopened.Get(RollY)), Is.EqualTo("derived-from-important")); + } + + // v1 reads v2's k through v2's entry, which lists v1's id: the list is the writer's statement, not v1's declaration, so to v1 + // k is read-only. v1's write of k is refused typed before anything is staged, the file is left byte for byte, and a v2 + // cache's condition on k still holds. + [Test] + public void AnOlderVersionCannotWriteANewerRecordItReadsThroughTheNewerEntry([Values] bool commit) + { + WriteImportant(); + var before = File.ReadAllBytes(RollPath); + using var newer = OpenRoll(RollV2); + var current = newer.Get(RollK)!; + + using (var older = OpenRoll(RollV1)) + { + var k = older.Get(RollK)!; + Assert.That(EmittedDocumentTypes.Read(k, "Name"), Is.EqualTo("k"), "v1 reads k"); + var refusal = Assert.Throws(() => + { + if (commit) + older.Commit(batch => batch.Upsert(RollV1, k, RollK)); + else + older.UpsertAsync(RollV1, k, RollK).GetAwaiter().GetResult(); + })!; + Assert.That((refusal.RecordKey, refusal.SchemaId), Is.EqualTo((RollK.Value, Read(RollPath).Records.Single().SchemaId))); + Assert.That(older.BackingStores.Single().IsDirty, Is.False, "nothing was staged"); + older.FlushAsync().GetAwaiter().GetResult(); + } + + Assert.That(File.ReadAllBytes(RollPath), Is.EqualTo(before)); + Assert.That(Derive(newer, current, unchangedOnly: false), Is.EqualTo(CultCommitOutcome.Committed)); + using var reopened = OpenRoll(RollV2); + Assert.That(Extra(reopened.Get(RollK)), Is.EqualTo("important")); + Assert.That(Extra(reopened.Get(RollY)), Is.EqualTo("derived-from-important")); + } + + // A record a write copies keeps its storedAt, so a condition another cache holds on it survives an unrelated write. + [TestCase(false)] + [TestCase(true)] + public void AnUnrelatedWriteKeepsTheStoredAtOfARecordItCopies(bool viaFlush) + { + WriteImportant(); + var written = Read(RollPath).Records.Single(); + using var held = OpenRoll(RollV2); + var current = held.Get(RollK)!; + + UnrelatedWrite(RollV2, viaFlush); + + Assert.That(Read(RollPath).Records.Single().StoredAt, Is.EqualTo(written.StoredAt)); + Assert.That(Derive(held, current, unchangedOnly: false), Is.EqualTo(CultCommitOutcome.Committed)); + } + + // The migration CompatibleSchemaIds exists for: v2 declares v1's id and v1's class stays registered beside it, in either + // order. A record under v1's id is v1's, its owner; one under v2's id is v2's. + [TestCase(false)] + [TestCase(true)] + public void AVersionKeptBesideTheVersionDeclaringItsIdReadsItsOwnRecords(bool reversed) + { + using (var older = OpenRoll(RollV1)) + older.Commit(batch => batch.Upsert(RollV1, New(RollV1, ("Name", "k")), RollK)); + using (var newer = OpenRoll(RollV2)) + newer.Commit(batch => + { + batch.Expect(RollY, null); + batch.Upsert(RollV2, New(RollV2, ("Name", "y"), ("Extra", "e")), RollY); + }); + Assert.That(Read(RollPath).Records.Select(record => record.SchemaId), Is.EquivalentTo(new[] { RollV1Id, CultDocumentRegistry.ForTypes(new[] { RollV2 }).GetRequired(RollV2).SchemaId })); + + using var both = Open(RollPath, CultDocumentRegistry.ForTypes(reversed ? new[] { RollV2, RollV1 } : new[] { RollV1, RollV2 })); + Assert.That(both.Get(RollK)!.GetType(), Is.EqualTo(RollV1)); + Assert.That(both.Get(RollY)!.GetType(), Is.EqualTo(RollV2)); + } + + [Test] + public void ADeclarationIsRegisteredWithoutItsOwnIdAndItsEntryListsTheOwnIdFirst() + { + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(Owner) }); + var descriptor = registry.GetRequired(); + + Assert.That(descriptor.SchemaId, Is.EqualTo(OwnerId), "the pinned id is this schema's id"); + Assert.That(descriptor.CompatibleSchemaIds, Is.Empty, "a type's own id and a blank id are not declared compatible ids"); + Assert.That(descriptor.ToCatalogEntry().CompatibleSchemaIds, Is.EqualTo(new[] { OwnerId })); + } + } +} diff --git a/tests/GameCult.Caching.Tests/ConditionalCommitTests.cs b/tests/GameCult.Caching.Tests/ConditionalCommitTests.cs index 5807b68e..137bebb5 100644 --- a/tests/GameCult.Caching.Tests/ConditionalCommitTests.cs +++ b/tests/GameCult.Caching.Tests/ConditionalCommitTests.cs @@ -83,18 +83,18 @@ namespace GameCult.Caching.Tests } [Test] - public void RecordRewrittenUnderAnotherSchemaWithItsStoredAtFailsExpect() + public void RecordRewrittenUnderAnotherSchemaFailsExpect() { SeedDrift(directory: false); using var cache = Drift(false, NarrowDrift).Open(); var observed = cache.Get(DriftKey)!; - var storedAt = cache.GetStored(DriftKey)!.StoredAt; using (var other = Drift(false, ReusedDrift).Open()) + { + other.BackingStores[0].Push(other.GetStored(DriftKey)!); other.BackingStores[0].PushAll(); - using (var reader = Drift(false, ReusedDrift).Open()) - Assert.That(reader.GetStored(DriftKey)!.StoredAt, Is.EqualTo(storedAt), "the rewrite kept storedAt"); + } - Assert.That(DriftCommit(cache, observed, 2), Is.False, "a record rewritten under another schema passed Expect on storedAt alone"); + Assert.That(DriftCommit(cache, observed, 2), Is.False, "a record rewritten under another schema passed Expect"); } // The directory store rewrites only what is pushed to it, so the other writer pushes the record it read unchanged. @@ -137,51 +137,10 @@ namespace GameCult.Caching.Tests cache.BackingStores[0].PushAll(); var stored = cache.GetStored(DriftKey)!; - Assert.That(stored.PersistedSchemaId, Is.EqualTo(stored.Descriptor.SchemaId), "the rewrite was not recorded"); + Assert.That(stored.StoredSchemaId, Is.EqualTo(stored.Descriptor.SchemaId), "the rewrite was not recorded"); Assert.That(DriftCommit(cache, cache.Get(DriftKey)!, 2), Is.True, "Expect failed after this cache's own push rewrote the record"); } - // A store written outside the library marks its own rewrite through the base class. - [Test] - public void ACustomStoreMarksItsOwnWholeFileRewrite() - { - SeedDrift(directory: false); - using var cache = new CultCache(NarrowDrift); - var store = new WholeFileStore(PathOf("drift.cc")); - cache.AddBackingStore(store); - store.PullAll(); - var loaded = cache.GetStored(DriftKey)!; - Assert.That(loaded.PersistedSchemaId, Is.Not.EqualTo(loaded.Descriptor.SchemaId), "the record was not loaded under drift"); - - store.Push(loaded); - store.PushAll(); - - Assert.That(cache.GetStored(DriftKey)!.PersistedSchemaId, Is.EqualTo(loaded.Descriptor.SchemaId), "the custom store's rewrite was not recorded"); - Assert.That(DriftCommit(cache, cache.Get(DriftKey)!, 2), Is.True, "Expect failed after the custom store rewrote the record"); - using var reader = Drift(false, NarrowDrift).Open(); - Assert.That(reader.Get(DriftKey)!.Value, Is.EqualTo(2)); - } - - // A store marks only what it holds: another cache's copy of the same record is refused and left as read. - [Test] - public void ACustomStoreCannotMarkAnEntryItDoesNotHold() - { - SeedDrift(directory: false); - using var owner = new CultCache(NarrowDrift); - var ownerStore = new WholeFileStore(PathOf("drift.cc")); - owner.AddBackingStore(ownerStore); - ownerStore.PullAll(); - using var other = new CultCache(NarrowDrift); - var otherStore = new WholeFileStore(PathOf("drift.cc")); - other.AddBackingStore(otherStore); - otherStore.PullAll(); - var foreign = owner.GetStored(DriftKey)!; - var persisted = foreign.PersistedSchemaId; - - Assert.Throws(() => otherStore.Mark(foreign), "a store marked another cache's entry"); - Assert.That(foreign.PersistedSchemaId, Is.EqualTo(persisted), "a refused mark changed the entry"); - } - [Test] public void OwnUnconditionalCommitKeepsDriftedExpectHolding() { @@ -215,12 +174,12 @@ namespace GameCult.Caching.Tests wide = writer.GetStored(DriftKey)!.Descriptor.SchemaId; using var cache = Drift(false, NarrowDrift).Open(); var loaded = cache.GetStored(DriftKey)!; - Assert.That(loaded.PersistedSchemaId, Is.EqualTo(wide), "the entry carries the schema id its record is stored under"); + Assert.That(loaded.StoredSchemaId, Is.EqualTo(wide), "the entry carries the schema id its record is stored under"); Assert.That(loaded.Descriptor.SchemaId, Is.Not.EqualTo(wide)); Assert.That(DriftCommit(cache, cache.Get(DriftKey)!, 2), Is.True); var written = cache.GetStored(DriftKey)!; - Assert.That(written.PersistedSchemaId, Is.EqualTo(written.Descriptor.SchemaId)); + Assert.That(written.StoredSchemaId, Is.EqualTo(written.Descriptor.SchemaId)); Assert.That(DriftCommit(cache, cache.Get(DriftKey)!, 3), Is.True, "Expect failed on the record this cache just wrote"); } @@ -429,6 +388,24 @@ namespace GameCult.Caching.Tests Assert.That(reads, Is.GreaterThan(0)); } + // With no backing store a record's version is minted with it: an Expect on the held record commits, and once the record is + // replaced after it was observed (here by the same document again) the Expect no longer holds. + [Test] + public void InMemoryExpectHoldsUntilTheRecordIsReplaced() + { + using var cache = new CultCache(CultDocumentRegistry.ForTypes(new[] { typeof(Counter) })); + cache.Commit(batch => batch.Upsert(new Counter { Name = "counter", Value = 1 }, new CultRecordHandle(Key))); + var first = cache.Get(Key)!; + + Assert.That(cache.Commit(batch => batch.Expect(Key, first)), Is.True); + + Assert.That(cache.Commit(batch => + { + batch.Expect(Key, first); + cache.UpsertAsync(first, new CultRecordHandle(Key)).GetAwaiter().GetResult(); + }), Is.False); + } + [Test] public void StoredAtIsStrictlyIncreasingPerKey() { @@ -775,7 +752,7 @@ namespace GameCult.Caching.Tests } [Test] - public void UnconditionalCommitIsLastWriterWinsLikeFlush() + public void UnconditionalCommitIsLastWriterWinsPerKeyLikeFlush() { var committed = LastWriter(SingleFile(PathOf("commit.cc")), (cache, key) => Assert.That(cache.Commit(batch => batch.Upsert(new Counter { Name = key.Value }, new CultRecordHandle(key))), Is.True)); @@ -785,10 +762,10 @@ namespace GameCult.Caching.Tests cache.FlushAllBackingStores(); }); - Assert.That(committed, Is.EqualTo(new[] { "a", "x" }), "an unconditional commit kept a record its cache never saw"); + Assert.That(committed, Is.EqualTo(new[] { "a", "c", "x" }), "an unconditional commit dropped a record another writer added"); Assert.That(committed, Is.EqualTo(flushed)); - // A directory store's flush lands staged keys onto the current manifest, and its commit does exactly the same. + // A directory store's flush lands staged keys onto the current manifest, and its commit does exactly the same: so does a single file's. var directoryCommitted = LastWriter(DirectoryStore(PathOf("commit-dir.cc")), (cache, key) => Assert.That(cache.Commit(batch => batch.Upsert(new Counter { Name = key.Value }, new CultRecordHandle(key))), Is.True)); var directoryFlushed = LastWriter(DirectoryStore(PathOf("flush-dir.cc")), (cache, key) => @@ -830,9 +807,11 @@ namespace GameCult.Caching.Tests writer.FlushAllBackingStores(); } + // A record of a schema this cache does not register is not corruption: the store opens and carries it (ForeignRecordTests). var bytes = File.ReadAllBytes(unknown); using var narrow = new CultCache(CultDocumentRegistry.ForTypes(new[] { typeof(Counter) })); - Assert.That(() => narrow.AddBackingStore(new SingleFileMessagePackBackingStore(unknown)), Throws.Exception, "a store with an unknown schema opened"); + narrow.AddBackingStore(new SingleFileMessagePackBackingStore(unknown)); + Assert.That(narrow.BackingStores.Single().ForeignRecords, Has.Count.EqualTo(1)); Assert.That(File.ReadAllBytes(unknown), Is.EqualTo(bytes), "opening rewrote a store with an unknown schema"); } @@ -1029,6 +1008,10 @@ namespace GameCult.Caching.Tests private static StoreFactory DirectoryStore(string path, CultDocumentRegistry? registry = null) => new(path, () => new DirectoryMessagePackBackingStore(path), registry); + // The id Wide is stored under, which the later versions declare compatible: a cache reads and writes a record under another + // id only when its type declares it. SeedDrift checks that the constant is Wide's id. + private const string WideDriftSchemaId = "sha256:6033aeca8a784899aff1f8c43eb9df6bf3364e4fb6014e423c65afd51a9b8f18"; + private static readonly CultRecordKey DriftKey = new("drift"); private static readonly CultDocumentRegistry WideDrift = CultDocumentRegistry.ForTypes(new[] { typeof(ConditionalDriftWide) }); private static readonly CultDocumentRegistry NarrowDrift = CultDocumentRegistry.ForTypes(new[] { typeof(ConditionalDriftNarrow) }); @@ -1040,6 +1023,7 @@ namespace GameCult.Caching.Tests // The record at DriftKey, written under v1. private void SeedDrift(bool directory) { + Assert.That(WideDrift.GetRequired(typeof(ConditionalDriftWide)).SchemaId, Is.EqualTo(WideDriftSchemaId)); using var cache = Drift(directory, WideDrift).Open(); Assert.That(cache.Commit(batch => batch.Upsert( new ConditionalDriftWide { Name = "drift", Value = 1, Extra = "extra" }, @@ -1052,65 +1036,6 @@ namespace GameCult.Caching.Tests batch.Upsert(new ConditionalDriftNarrow { Name = "drift", Value = value }, new CultRecordHandle(DriftKey)); }); - // A store derived from CacheBackingStore alone, as a consumer would write one: it rewrites the whole file on every - // write, in the single-file MessagePack format, and marks each record it rewrote. - private sealed class WholeFileStore : CacheBackingStore - { - private readonly string _path; - - public WholeFileStore(string path) => _path = path; - - private CultPersistedStoreSnapshot Read() => CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(_path)); - - public override void PullAll() => Held(() => - { - var snapshot = Read(); - var loaded = snapshot.Records - .Select(record => ToStoredDocument(record, snapshot.SchemaCatalog, - (type, payload) => CultDocumentMessagePackSerialization.DeserializeUntyped(type, payload, Registry), out _)) - .ToArray(); - Loaded?.Invoke(loaded, Array.Empty()); - foreach (var stored in loaded) - Entries[stored.Key.Value] = stored; - }); - - public override void Push(CultStoredDocument entry) => Held(() => - { - Entries[entry.Key.Value] = entry; - IsDirty = true; - }); - - public override void Delete(CultStoredDocument entry) => throw new NotSupportedException(); - - public void Mark(CultStoredDocument entry) => MarkWritten(entry); - - public override void PushAll() => Held(Write); - - public override CultCommitOutcome CommitBatch(CultCommitRequest request, bool wait) => Held(() => - { - if (!request.ConditionsHold(Read().Records, Entries.Values)) - return CultCommitOutcome.Mismatch; - foreach (var entry in request.Upserts) - Entries[entry.Key.Value] = entry; - Write(); - return CultCommitOutcome.Committed; - }); - - private void Write() - { - var snapshot = new CultPersistedStoreSnapshot - { - Records = Entries.Values.Select(entry => ToPersistedRecord(entry, - document => CultDocumentMessagePackSerialization.SerializeUntyped(document, document.GetType(), Registry))).ToArray(), - SchemaCatalog = Entries.Values.Select(entry => entry.Descriptor.ToCatalogEntry()).ToArray() - }; - File.WriteAllBytes(_path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); - foreach (var entry in Entries.Values) - MarkWritten(entry); - MarkFlushSucceeded(); - } - } - // Pauses a pull between reading the file and handing the records to the cache. private sealed class PausingStore : SingleFileMessagePackBackingStore { @@ -1130,9 +1055,9 @@ namespace GameCult.Caching.Tests base.Push(entry); } - protected override CultPersistedStoreSnapshot DeserializeSnapshot(byte[] data) + protected override (CultPersistedStoreSnapshot Snapshot, byte[][] CatalogBytes, byte[][] RecordBytes) ReadStore(byte[] data) { - var snapshot = base.DeserializeSnapshot(data); + var snapshot = base.ReadStore(data); if (Pause is { } pause) { Paused = true; @@ -1196,7 +1121,7 @@ namespace GameCult.Caching.Tests public string Extra = string.Empty; } - [CultDocument("tests.conditional_drift", "tests.conditional_drift.v2")] + [CultDocument("tests.conditional_drift", "tests.conditional_drift.v2", CompatibleSchemaIds = new[] { WideDriftSchemaId })] [MessagePackObject(AllowPrivate = true)] internal sealed class ConditionalDriftNarrow { @@ -1208,7 +1133,7 @@ namespace GameCult.Caching.Tests public int Value; } - [CultDocument("tests.conditional_drift", "tests.conditional_drift.v3")] + [CultDocument("tests.conditional_drift", "tests.conditional_drift.v3", CompatibleSchemaIds = new[] { WideDriftSchemaId })] [MessagePackObject(AllowPrivate = true)] internal sealed class ConditionalDriftReused { diff --git a/tests/GameCult.Caching.Tests/CultDocumentRegistryTests.cs b/tests/GameCult.Caching.Tests/CultDocumentRegistryTests.cs index 2e061f47..5f60544d 100644 --- a/tests/GameCult.Caching.Tests/CultDocumentRegistryTests.cs +++ b/tests/GameCult.Caching.Tests/CultDocumentRegistryTests.cs @@ -1,9 +1,8 @@ using System; using System.Linq; -using System.Reflection; -using System.Reflection.Emit; using System.Threading.Tasks; using NUnit.Framework; +using static GameCult.Caching.Tests.EmittedDocumentTypes; namespace GameCult.Caching.Tests; @@ -13,8 +12,8 @@ public sealed class CultDocumentRegistryTests [Test] public void ForTypesContainsOnlyExplicitDocuments() { - var requested = CreateDocumentType("explicit", "tests.registry.explicit", "v1"); - var unrelated = CreateDocumentType("unrelated", "tests.registry.unrelated", "v1"); + var requested = Emit("explicit", "tests.registry.explicit", "v1"); + var unrelated = Emit("unrelated", "tests.registry.unrelated", "v1"); var registry = CultDocumentRegistry.ForTypes(new[] { requested, requested }); @@ -26,7 +25,7 @@ public sealed class CultDocumentRegistryTests public void IdenticalTypeRegistrationIsIdempotent() { var registry = new CultDocumentRegistry(); - var type = CreateDocumentType("idempotent", "tests.registry.idempotent", "v1"); + var type = Emit("idempotent", "tests.registry.idempotent", "v1"); var first = registry.GetRequired(type); var second = registry.GetRequired(type); @@ -34,34 +33,12 @@ public sealed class CultDocumentRegistryTests Assert.That(second, Is.SameAs(first)); } - [Test] - public void ExactWireCompatibleAliasRegistersByTypeWithOneCanonicalSchemaDescriptor() - { - var registry = new CultDocumentRegistry(); - var firstType = CreateDocumentType("alias_owner", "tests.registry.alias", "v1"); - var aliasType = CreateDocumentType("alias_claimant", "tests.registry.alias", "v1"); - - var canonical = registry.GetRequired(firstType); - var alias = registry.GetRequired(aliasType); - - Assert.Multiple(() => - { - Assert.That(alias.DocumentType, Is.EqualTo(aliasType)); - Assert.That(alias.SchemaId, Is.EqualTo(canonical.SchemaId)); - Assert.That(registry.GetRequired(aliasType), Is.SameAs(alias)); - Assert.That(registry.GetRequiredBySchemaId(canonical.SchemaId), Is.SameAs(canonical)); - Assert.That( - registry.AllDescriptors.Count(descriptor => descriptor.SchemaId == canonical.SchemaId), - Is.EqualTo(2)); - }); - } - [Test] public void DifferentVersionsSharingSchemaNameRegisterWithoutReflectionOrderSelection() { var registry = new CultDocumentRegistry(); - var firstType = CreateDocumentType("version_one", "tests.registry.versioned", "v1"); - var secondType = CreateDocumentType("version_two", "tests.registry.versioned", "v2"); + var firstType = Emit("version_one", "tests.registry.versioned", "v1"); + var secondType = Emit("version_two", "tests.registry.versioned", "v2"); var first = registry.GetRequired(firstType); var second = registry.GetRequired(secondType); @@ -86,33 +63,11 @@ public sealed class CultDocumentRegistryTests .And.Message.Contains("v2")); } - [Test] - public void IncompatibleTypesClaimingSameSchemaNameAndVersionFailBeforeMutation() - { - var registry = new CultDocumentRegistry(); - var firstType = CreateDocumentType("layout_owner", "tests.registry.layout_collision", "v1"); - var secondType = CreateDocumentType( - "layout_claimant", - "tests.registry.layout_collision", - "v1", - addStringField: true); - var first = registry.GetRequired(firstType); - - Assert.That( - () => registry.GetRequired(secondType), - Throws.TypeOf() - .With.Message.Contains("schema name 'tests.registry.layout_collision' version 'v1'") - .And.Message.Contains(firstType.FullName) - .And.Message.Contains(secondType.FullName)); - Assert.That(registry.GetRequiredBySchemaId(first.SchemaId), Is.SameAs(first)); - Assert.That(registry.AllDescriptors.Any(descriptor => descriptor.DocumentType == secondType), Is.False); - } - [Test] public void ConcurrentRefreshAndReadsObserveCompleteRegistrySnapshots() { var registry = new CultDocumentRegistry(); - var type = CreateDocumentType("refresh_visible", "tests.registry.refresh_visible", "v1"); + var type = Emit("refresh_visible", "tests.registry.refresh_visible", "v1"); var descriptor = registry.GetRequired(type); Parallel.For(0, 20, _ => @@ -123,33 +78,94 @@ public sealed class CultDocumentRegistryTests }); } - private static Type CreateDocumentType( - string typeName, - string schemaName, - string schemaVersion, - bool addStringField = false) + // A second type for one schema is refused, typed, naming both and the claimed id, and the registry keeps what it had. + private static void AssertSecondClaimRefused(Type first, Type second) { - var assembly = AssemblyBuilder.DefineDynamicAssembly( - new AssemblyName("GameCult.Caching.RegistryTests." + Guid.NewGuid().ToString("N")), - AssemblyBuilderAccess.Run); - var module = assembly.DefineDynamicModule("main"); - var type = module.DefineType( - typeName, - TypeAttributes.Class | TypeAttributes.Public | TypeAttributes.Sealed); - var attributeConstructor = typeof(CultDocumentAttribute).GetConstructor(new[] { typeof(string), typeof(string) })!; - type.SetCustomAttribute(new CustomAttributeBuilder( - attributeConstructor, - new object[] { schemaName, schemaVersion })); - type.SetCustomAttribute(new CustomAttributeBuilder( - typeof(global::MessagePack.MessagePackObjectAttribute).GetConstructor(new[] { typeof(bool) })!, - new object[] { false })); - if (addStringField) + var schemaId = CultDocumentRegistry.ForTypes(new[] { second }).GetRequired(second).SchemaId; + var refusal = Assert.Throws(() => CultDocumentRegistry.ForTypes(new[] { first, second }))!; + Assert.Multiple(() => { - type.DefineField("Value", typeof(string), FieldAttributes.Public).SetCustomAttribute(new CustomAttributeBuilder( - typeof(global::MessagePack.KeyAttribute).GetConstructor(new[] { typeof(int) })!, - new object[] { 0 })); - } + Assert.That(refusal.SchemaId, Is.EqualTo(schemaId)); + Assert.That(refusal.Message, Does.Contain(first.FullName).And.Contain(second.FullName)); + }); - return type.CreateType()!; + var registry = CultDocumentRegistry.ForTypes(new[] { first }); + Assert.That(() => registry.GetRequired(second), Throws.TypeOf()); + Assert.That(registry.AllDescriptors.Select(descriptor => descriptor.DocumentType), Is.EqualTo(new[] { first })); + } + + private static CultSchemaCatalogEntry[] CatalogOf(params Type[] types) => + types.Select(type => CultDocumentRegistry.ForTypes(new[] { type }).GetRequired(type).ToCatalogEntry()).ToArray(); + + // Owner beats lister: a type that owns an id and one that lists it register together in either order (the v1 class kept + // beside a v2 class declaring v1's id). A record under the id resolves to its owner; one under the lister's own id to the + // lister; without the owner registered, the id resolves to the lister. + [TestCase(false)] + [TestCase(true)] + public void AnOwnerAndATypeListingItsIdRegisterTogetherAndTheOwnerWins(bool reversed) + { + var owner = Emit("owns", "tests.registry.owns", "v1"); + var ownerId = CultDocumentRegistry.ForTypes(new[] { owner }).GetRequired(owner).SchemaId; + var lister = Emit("lists", "tests.registry.owns", "v2", new[] { new Field("Extra", typeof(string), 0) }, new[] { ownerId }); + var listerId = CultDocumentRegistry.ForTypes(new[] { lister }).GetRequired(lister).SchemaId; + var catalog = CatalogOf(owner, lister); + + var registry = CultDocumentRegistry.ForTypes(reversed ? new[] { lister, owner } : new[] { owner, lister }); + + Assert.Multiple(() => + { + Assert.That(registry.AllDescriptors.Select(descriptor => descriptor.DocumentType), Is.EquivalentTo(new[] { owner, lister })); + Assert.That(registry.ResolvePersistedSchema(ownerId, catalog).DocumentType, Is.EqualTo(owner)); + Assert.That(registry.ResolvePersistedSchema(listerId, catalog).DocumentType, Is.EqualTo(lister)); + Assert.That(CultDocumentRegistry.ForTypes(new[] { lister }).ResolvePersistedSchema(ownerId, catalog).DocumentType, Is.EqualTo(lister)); + }); + } + + // Two types listing one id register together in either order. With an owner registered, a record under the id is the + // owner's; with none, the id names no single type, and a record under it is refused, typed, naming the listers. + [TestCase(false)] + [TestCase(true)] + public void TwoTypesListingOneIdRegisterAndARecordUnderItNeedsAnOwner(bool reversed) + { + var owner = Emit("shared_owner", "tests.registry.shared", "v1"); + var sharedId = CultDocumentRegistry.ForTypes(new[] { owner }).GetRequired(owner).SchemaId; + var a = Emit("declares_a", "tests.registry.declares_a", "v1", compatibleSchemaIds: new[] { sharedId }); + var b = Emit("declares_b", "tests.registry.declares_b", "v1", compatibleSchemaIds: new[] { sharedId }); + var listers = reversed ? new[] { b, a } : new[] { a, b }; + var catalog = CatalogOf(owner); + + var ambiguous = CultDocumentRegistry.ForTypes(listers); + var refusal = Assert.Throws(() => ambiguous.ResolvePersistedSchema(sharedId, catalog))!; + Assert.Multiple(() => + { + Assert.That(refusal.SchemaId, Is.EqualTo(sharedId)); + Assert.That(refusal.SchemaNames, Is.EquivalentTo(new[] { "tests.registry.declares_a", "tests.registry.declares_b" })); + Assert.That(refusal.Message, Does.Contain(a.FullName).And.Contain(b.FullName)); + }); + + Assert.That(CultDocumentRegistry.ForTypes(listers.Append(owner)).ResolvePersistedSchema(sharedId, catalog).DocumentType, Is.EqualTo(owner)); + Assert.That(CultDocumentRegistry.ForTypes(listers.Prepend(owner)).ResolvePersistedSchema(sharedId, catalog).DocumentType, Is.EqualTo(owner)); + } + + // One type per schema, in every runtime: a second type of the same schema name and version is refused whichever registers + // first, whether it is identical to the first, declares different compatible ids, or has different members. + [TestCase("identical", false)] + [TestCase("identical", true)] + [TestCase("declaration", false)] + [TestCase("declaration", true)] + [TestCase("members", false)] + [TestCase("members", true)] + public void ASecondTypeForOneSchemaIsRefused(string difference, bool reversed) + { + var schemaName = $"tests.registry.claimed_twice_{difference}"; + var first = Emit($"first_{difference}", schemaName, "v1"); + var second = difference switch + { + "identical" => Emit($"second_{difference}", schemaName, "v1"), + "declaration" => Emit($"second_{difference}", schemaName, "v1", compatibleSchemaIds: new[] { "tests.registry.older" }), + _ => Emit($"second_{difference}", schemaName, "v1", new[] { new Field("Value", typeof(string), 0) }), + }; + + AssertSecondClaimRefused(reversed ? second : first, reversed ? first : second); } } diff --git a/tests/GameCult.Caching.Tests/CultMathExprVariantTests.cs b/tests/GameCult.Caching.Tests/CultMathExprVariantTests.cs new file mode 100644 index 00000000..181a7ff0 --- /dev/null +++ b/tests/GameCult.Caching.Tests/CultMathExprVariantTests.cs @@ -0,0 +1,79 @@ +#nullable enable +using System; +using System.IO; +using CultMath.Expr; +using GameCult.Caching; +using GameCult.Caching.MessagePack; +using MessagePack; +using NUnit.Framework; + +namespace GameCult.Caching.Tests +{ + // An ExprGraph is a plain value with no element-id lists. It survives a single-file cache, a variant inherits it, + // a variant may override it whole, and an edit to the base reaches the inheriting variant. + public class CultMathExprVariantTests + { + [CultDocument("tests.expr_variant", "tests.expr_variant.v1")] + [MessagePackObject(AllowPrivate = true)] + internal sealed class ExprDoc + { + [Key(0)] [CultName] public string Name = string.Empty; + [Key(1)] public ExprGraph Graph = default!; + [Key(2)] public float Scale; + } + + private static ExprGraph Graph(float k) + { + var b = new ExprBuilder(); + var x = b.Input("x"); + var p = b.Param("k", k); + b.Output("o", b.Call("mul", x, p)); + return b.Build(); + } + + private static byte[] Enc(ExprGraph g) => MessagePackSerializer.Serialize(g, CultDocumentMessagePackSerialization.Options); + + [Test] + public void ExprGraphRoundTripsThroughACacheAndIsInheritedOrOverriddenByVariants() + { + var dir = Path.Combine(Path.GetTempPath(), "cultlib-exprvariant-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(dir); + try + { + var path = Path.Combine(dir, "expr.cc"); + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(ExprDoc) }); + var baseKey = new CultRecordKey("base"); + var scaledKey = new CultRecordKey("scaled"); + var regraphedKey = new CultRecordKey("regraphed"); + + using (var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = registry })) + { + cache.Commit(b => b.Upsert(typeof(ExprDoc), new ExprDoc { Name = "base", Graph = Graph(1.5f), Scale = 2f }, baseKey)); + cache.Commit(b => b.UpsertVariant(scaledKey, baseKey, new[] { cache.Override(nameof(ExprDoc.Name), "scaled"), cache.Override(nameof(ExprDoc.Scale), 3f) })); + cache.Commit(b => b.UpsertVariant(regraphedKey, baseKey, new[] { cache.Override(nameof(ExprDoc.Name), "regraphed"), cache.Override(nameof(ExprDoc.Graph), Graph(4f)) })); + } + + using (var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = registry })) + { + var baseDoc = cache.Get(baseKey)!; + var scaled = cache.Get(scaledKey)!; + var regraphed = cache.Get(regraphedKey)!; + + Assert.That(Enc(baseDoc.Graph), Is.EqualTo(Enc(Graph(1.5f))), "the base graph survives the file"); + Assert.That(scaled.Scale, Is.EqualTo(3f)); + Assert.That(Enc(scaled.Graph), Is.EqualTo(Enc(Graph(1.5f))), "a variant that does not override the graph inherits it"); + Assert.That(Enc(regraphed.Graph), Is.EqualTo(Enc(Graph(4f))), "a variant that overrides the graph replaces it whole"); + Assert.That(regraphed.Scale, Is.EqualTo(2f), "and keeps the rest of the base"); + + cache.Commit(b => b.Upsert(typeof(ExprDoc), new ExprDoc { Name = "base", Graph = Graph(9f), Scale = 2f }, baseKey)); + Assert.That(Enc(cache.Get(scaledKey)!.Graph), Is.EqualTo(Enc(Graph(9f))), "an edit to the base reaches the inheriting variant"); + Assert.That(Enc(cache.Get(regraphedKey)!.Graph), Is.EqualTo(Enc(Graph(4f))), "and not the overriding one"); + } + } + finally + { + Directory.Delete(dir, recursive: true); + } + } + } +} diff --git a/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs b/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs new file mode 100644 index 00000000..2bc1d9df --- /dev/null +++ b/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs @@ -0,0 +1,1087 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Reflection; +using System.Text.RegularExpressions; +using GameCult.Caching; +using MessagePack; +using NUnit.Framework; + +namespace GameCult.Caching.Tests +{ + // The id engine's two halves, driven directly: which element types a document type reaches, and how unset ids are filled. + public class ElementIdEngineTests + { + private static List Problems(Type type) => CultElementIds.Inspect(type).Problems; + + private static int Assign(object root, string key, bool deterministic) + { + var plan = CultElementIds.Plan(root.GetType(), root, key, deterministic); + plan.Apply(); + return plan.Count; + } + + private static void RefusesBare(Type holder) => + Assert.That(Problems(holder), Has.Count.EqualTo(1).And.All.Contain("EBare").And.All.Contain("[CultElementId]"), holder.Name); + + [Test] + public void EveryShapeThatCanHoldAListReachesItsElements() + { + foreach (var holder in new[] + { + typeof(HList), typeof(HArray), typeof(HDictionary), typeof(HNested), typeof(HListOfLists), + typeof(HElementNest), typeof(HUnion), typeof(HNullableInner), typeof(HStruct), typeof(HStructList), typeof(HNullableStruct), + typeof(HReadOnlyMap) + }) + RefusesBare(holder); + Assert.That(Problems(typeof(HFine)), Is.Empty); + } + + [Test] + public void EachWayAnIdCanBeWrongIsNamed() + { + Assert.That(Problems(typeof(HTwo)).Single(), Does.Contain("two [CultElementId] members").And.Contain("A").And.Contain("B")); + Assert.That(Problems(typeof(HInt)).Single(), Does.Contain("not a string").And.Contain("Id")); + Assert.That(Problems(typeof(HNoKey)).Single(), Does.Contain("no integer [Key]").And.Contain("Id")); + Assert.That(Problems(typeof(HGetOnly)).Single(), Does.Contain("cannot be assigned").And.Contain("Id")); + Assert.That(Problems(typeof(HMissingSource)).Single(), Does.Contain("Nope").And.Contain("not a keyed member")); + Assert.That(Problems(typeof(HBadSourceKind)).Single(), Does.Contain("Tags").And.Contain("not a string or a number")); + Assert.That(Problems(typeof(HUnkeyedSource)).Single(), Does.Contain("Loose").And.Contain("not a keyed member")); + Assert.That(Problems(typeof(HNullableSource)), Is.Empty, "a nullable number is a number"); + } + + [Test] + public void AssignFillsEveryListShapeAndOnlyTheUnsetOnes() + { + var root = new HAll + { + Dict = { ["k"] = new HHasIds { Items = { new EIded(), new EIded { Id = "0123456789ab" } } } }, + Grid = { new List { new EIded(), new EIded() } }, + Nest = { new ENest { Sub = { new EIded() } } }, + }; + var count = Assign(root, "r", deterministic: false); + Assert.That(count, Is.EqualTo(5)); + Assert.That(root.Dict["k"].Items[1].Id, Is.EqualTo("0123456789ab")); + var ids = new[] { root.Dict["k"].Items[0].Id }.Concat(root.Grid[0].Select(x => x.Id)).Concat(new[] { root.Nest[0].Id, root.Nest[0].Sub[0].Id }).ToArray(); + Assert.That(ids, Has.All.Matches(id => id.Length == 12)); + Assert.That(ids.Distinct().Count(), Is.EqualTo(ids.Length)); + Assert.That(Assign(root, "r", deterministic: false), Is.EqualTo(0), "a second pass finds nothing unset"); + } + + // Holds is what a store marks its header by: an id anywhere a list can sit counts, and an unset one does not. + [Test] + public void HoldsFindsASetIdWhereverAListCanSitAndNothingElse() + { + const string id = "0123456789ab"; + Assert.That(CultElementIds.Holds(typeof(HAll), null), Is.False); + Assert.That(CultElementIds.Holds(typeof(HAll), new HAll + { + Dict = { ["k"] = new HHasIds { Items = { new EIded() } } }, + Grid = { new List { new EIded() } }, + Nest = { new ENest { Sub = { new EIded() } } }, + }), Is.False, "unset ids hold nothing"); + Assert.That(CultElementIds.Holds(typeof(HAll), new HAll { Dict = { ["k"] = new HHasIds { Items = { new EIded { Id = id } } } } }), Is.True, "a dictionary value"); + Assert.That(CultElementIds.Holds(typeof(HAll), new HAll { Grid = { new List { new EIded { Id = id } } } }), Is.True, "a list of lists"); + Assert.That(CultElementIds.Holds(typeof(HAll), new HAll { Nest = { new ENest { Sub = { new EIded { Id = id } } } } }), Is.True, "a list under an unset element"); + Assert.That(CultElementIds.Holds(typeof(HAll), new HAll { Nest = { new ENest { Id = id } } }), Is.True, "an element"); + Assert.That(CultElementIds.Holds(typeof(HAll), new HAll { Dict = { ["a"] = new HHasIds { Items = { new EIded() } }, ["b"] = new HHasIds { Items = { new EIded { Id = id } } } } }), Is.True, "the second dictionary value"); + Assert.That(CultElementIds.Holds(typeof(HAll), new HAll { Grid = { new List { new EIded() }, new List { new EIded { Id = id } } } }), Is.True, "the second list"); + Assert.That(CultElementIds.Holds(typeof(HAll), new HAll { Nest = { new ENest(), new ENest { Sub = { new EIded { Id = id } } } } }), Is.True, "the second item of a list of elements"); + Assert.That(CultElementIds.Holds(typeof(HAll), new HAll { Grid = { new List { new EIded(), new EIded { Id = id } } } }), Is.True, "the second element of a list"); + } + + [Test] + public void APlanCountsAndChangesNothingUntilApplied() + { + var root = new HHasIds { Items = { new EIded(), new EIded() } }; + var plan = CultElementIds.Plan(typeof(HHasIds), root, "r", deterministic: true); + Assert.That(plan.Count, Is.EqualTo(2)); + Assert.That(root.Items.Select(x => x.Id), Has.All.Empty); + plan.Apply(); + Assert.That(root.Items.Select(x => x.Id), Has.All.Not.Empty); + plan.Undo(); + Assert.That(root.Items.Select(x => x.Id), Has.All.Empty); + } + + [Test] + public void ADeterministicIdDependsOnTheRecordAndThePositionAndNothingElse() + { + HHasIds Make() => new() { Items = { new EIded(), new EIded() } }; + string[] Ids(string key) + { + var root = Make(); + Assign(root, key, deterministic: true); + return root.Items.Select(x => x.Id).ToArray(); + } + + Assert.That(Ids("a"), Is.EqualTo(Ids("a"))); + Assert.That(Ids("a"), Is.Not.EqualTo(Ids("b"))); + Assert.That(Ids("a")[0], Is.Not.EqualTo(Ids("a")[1])); + Assert.That(Ids("a"), Has.All.Matches(id => Regex.IsMatch(id, "^[0-9a-f]{12}$"))); + var random = Make(); + Assign(random, "a", deterministic: false); + Assert.That(random.Items.Select(x => x.Id), Is.Not.EqualTo(Ids("a")), "a write mints random ids, a load mints from the path"); + } + + [Test] + public void ADuplicateIsRefusedInAnyListShapeNamingTheRecordTheListAndTheId() + { + var dict = new HAll { Dict = { ["k"] = new HHasIds { Items = { new EIded { Id = "aaaaaaaaaaaa" }, new EIded { Id = "aaaaaaaaaaaa" } } } } }; + var refusal = Assert.Throws(() => Assign(dict, "rec", deterministic: true))!; + Assert.That(refusal.RecordKey, Is.EqualTo("rec")); + Assert.That(refusal.ElementId, Is.EqualTo("aaaaaaaaaaaa")); + Assert.That(refusal.ListPath, Is.EqualTo("rec.0{k}.0")); + Assert.That(refusal.Message, Does.Contain("rec").And.Contain("aaaaaaaaaaaa").And.Contain("rec.0{k}.0")); + var grid = new HAll { Grid = { new List { new EIded { Id = "bbbbbbbbbbbb" }, new EIded { Id = "bbbbbbbbbbbb" } } } }; + Assert.Throws(() => Assign(grid, "r", deterministic: false)); + } + + [Test] + public void TheHexRefusalNamesTheDerivedFormForReadableIds() + { + var refusal = Assert.Throws(() => Assign(new HHasIds { Items = { new EIded { Id = "root" } } }, "r", deterministic: false))!; + Assert.That(refusal.Message, Does.Contain("12 lowercase hex").And.Contain("[CultElementId(nameof(Member))]")); + } + + [Test] + public void ARandomIdMustBeTwelveLowercaseHexCharactersAndADerivedIdNeedOnlyBeText() + { + foreach (var bad in new[] { "x", "ABCDEF012345", "0123456789a", "0123456789abc", "0123456789ag", "0123456789 a" }) + { + var refusal = Assert.Throws(() => Assign(new HHasIds { Items = { new EIded { Id = bad } } }, "r", deterministic: false), bad)!; + Assert.That(refusal.Message, Does.Contain("12 lowercase hex").And.Contain(bad), bad); + Assert.That(refusal.ElementId, Is.EqualTo(bad)); + } + + Assert.That(Assign(new HHasIds { Items = { new EIded { Id = "0123456789ab" }, new EIded { Id = "abcdef012345" } } }, "r", deterministic: false), Is.EqualTo(0)); + Assert.That(Assign(new HDerived { Items = { new EDerived { Number = 5, Id = "not hex, not the source" } } }, "r", deterministic: false), Is.EqualTo(0)); + } + + [Test] + public void ADerivedIdWhoseSourceIsNullOrEmptyIsRefusedNamingTheSourceMember() + { + foreach (var name in new string?[] { null, "" }) + { + var refusal = Assert.Throws( + () => Assign(new HDerivedText { Items = { new EDerivedText { Name = "fine" }, new EDerivedText { Name = name } } }, "rec", deterministic: false), + name ?? "null")!; + Assert.That(refusal.Member, Is.EqualTo("Name")); + Assert.That(refusal.RecordKey, Is.EqualTo("rec")); + Assert.That(refusal.Message, Does.Contain("Name").And.Contain("null or empty")); + } + + Assert.That(Assign(new HDerivedText { Items = { new EDerivedText { Name = "a" }, new EDerivedText { Name = "b" } } }, "r", deterministic: false), Is.EqualTo(2)); + } + + [Test] + public void APlanThatRefusesHasChangedNothingAndAnUndoRestoresWhatWasThere() + { + var root = new HAll + { + Dict = { ["k"] = new HHasIds { Items = { new EIded(), new EIded() } } }, + Grid = { new List { new EIded { Id = "cccccccccccc" }, new EIded { Id = "cccccccccccc" } } }, + }; + Assert.Throws(() => Assign(root, "r", deterministic: false)); + Assert.That(root.Dict["k"].Items.Select(x => x.Id), Has.All.Empty, "the list before the refused one was not minted into"); + + var ok = new HHasIds { Items = { new EIded(), new EIded { Id = "dddddddddddd" } } }; + var plan = CultElementIds.Plan(typeof(HHasIds), ok, "r", deterministic: false); + plan.Apply(); + plan.Undo(); + Assert.That(ok.Items.Select(x => x.Id), Is.EqualTo(new[] { "", "dddddddddddd" })); + } + + [Test] + public void ADeterministicIdIsTheSameUnderEveryCultureIncludingDictionaryKeys() + { + var expected = new[] + { + Sha6("r|r.0{1.5}.0/0|0"), Sha6("r|r.1{-3}.0/0|0"), Sha6("r|r.2{09/29/2026 12:00:00}.0/0|0") + }; + string[] Mint() + { + var root = new HCulture + { + ByDouble = { [1.5] = new HHasIds { Items = { new EIded() } } }, + ByInt = { [-3] = new HHasIds { Items = { new EIded() } } }, + ByDate = { [new DateTime(2026, 9, 29, 12, 0, 0, DateTimeKind.Unspecified)] = new HHasIds { Items = { new EIded() } } }, + }; + Assign(root, "r", deterministic: true); + return new[] { root.ByDouble[1.5].Items[0].Id, root.ByInt[-3].Items[0].Id, root.ByDate.Values.Single().Items[0].Id }; + } + + var saved = System.Globalization.CultureInfo.CurrentCulture; + try + { + foreach (var name in new[] { "en-US", "de-DE", "sv-SE", "ar-SA" }) + { + System.Globalization.CultureInfo.CurrentCulture = System.Globalization.CultureInfo.GetCultureInfo(name); + if (name == "de-DE") + Assert.That(1.5.ToString(), Is.EqualTo("1,5"), "the runtime has no culture data, so this test would prove nothing"); + Assert.That(Mint(), Is.EqualTo(expected), name); + } + } + finally + { + System.Globalization.CultureInfo.CurrentCulture = saved; + } + } + + [Test] + public void AnInterfaceUnionIsJudgedOnItsConcreteTypesAndMintedIntoThem() + { + Assert.That(Problems(typeof(HIfaceBad)).Single(), Does.Contain("IfaceBad").And.Contain("no [CultElementId] member")); + Assert.That(Problems(typeof(HIfaceGood)), Is.Empty); + var root = new HIfaceGood { Items = { new IfaceGood(), new IfaceGood() } }; + Assert.That(Assign(root, "r", deterministic: false), Is.EqualTo(2)); + Assert.That(root.Items.Select(x => ((IfaceGood)x).Id), Has.All.Not.Empty); + } + + [Test] + public void AnAbstractUnionBaseIsNotJudgedAndASubtypeListedTwiceIsJudgedOnce() + { + Assert.That(Problems(typeof(HDup)).Single(), Does.Contain("DupSub").And.Contain("no [CultElementId] member")); + Assert.That(Problems(typeof(HAbstractNoId)), Is.Empty, "the abstract base has no id, and no element can be one"); + } + + [Test] + public void ASubtypeThatReusesTheBaseIdsKeyIsRefusedAtRegistration() + { + var problem = Problems(typeof(HKeyClash)).Single(); + Assert.That(problem, Does.Contain("KeyClash").And.Contain("Id").And.Contain("Clash").And.Contain("Key(0)")); + } + + private static string Sha6(string text) + { + using var sha = System.Security.Cryptography.SHA256.Create(); + return string.Concat(sha.ComputeHash(System.Text.Encoding.UTF8.GetBytes(text)).Take(6).Select(b => b.ToString("x2"))); + } + + [Test] + public void TheDeterministicIdIsThePinnedDerivationAndARetryFollowsACollision() + { + var plain = new HHasIds { Items = { new EIded(), new EIded() } }; + Assign(plain, "r", deterministic: true); + Assert.That(plain.Items.Select(x => x.Id), Is.EqualTo(new[] { Sha6("r|r.0/0|0"), Sha6("r|r.0/1|0") })); + + var collide = new HHasIds { Items = { new EIded(), new EIded { Id = Sha6("r|r.0/0|0") } } }; + Assign(collide, "r", deterministic: true); + Assert.That(collide.Items[0].Id, Is.EqualTo(Sha6("r|r.0/0|1"))); + + var dict = new HAll { Dict = { ["k"] = new HHasIds { Items = { new EIded() } } } }; + Assign(dict, "r", deterministic: true); + Assert.That(dict.Dict["k"].Items[0].Id, Is.EqualTo(Sha6("r|r.0{k}.0/0|0"))); + } + + [Test] + public void ANullElementIsSkippedAndAMintedIdNeverEqualsAnExplicitOne() + { + var root = new HHasIds { Items = { null!, new EIded(), new EIded { Id = "abcdef012345" } } }; + Assert.That(Assign(root, "r", deterministic: false), Is.EqualTo(1)); + Assert.That(root.Items[1].Id, Is.Not.Empty.And.Not.EqualTo("explicit")); + } + + [Test] + public void ADerivedIdFollowsItsSourceIncludingANullableNumberAndDuplicatesAreRefused() + { + var root = new HDerived { Items = { new EDerived { Number = 5 }, new EDerived { Number = 12 }, new EDerived { Number = 5, Id = "own" } } }; + Assign(root, "r", deterministic: false); + Assert.That(root.Items.Select(x => x.Id), Is.EqualTo(new[] { "5", "12", "own" })); + var twice = new HDerived { Items = { new EDerived { Number = 5 }, new EDerived { Number = 5 } } }; + Assert.That(Assert.Throws(() => Assign(twice, "r", deterministic: false))!.Message, Does.Contain("'5'")); + } + + // A list element is written by its [Key] layout or its id never reaches disk. A union arm without [MessagePackObject] + // is written by some other formatter; a list of a type that is not keyed at all is a value list, replaced whole. + [Test] + public void ANonKeyedArmOfAListUnionIsRefusedAtRegistration() + { + var problem = Problems(typeof(HUnkeyedArmList)).Single(); + Assert.That(problem, Does.Contain("UnkeyedArm").And.Contain("formatter the cache does not own").And.Contain("HUnkeyedArmList")); + Assert.That(Problems(typeof(HUnkeyedArmField)), Is.Empty, "the same union as a field has no element id to lose"); + Assert.That(Problems(typeof(HValueList)), Is.Empty, "a list of a non-keyed type is a value list"); + } + + // [MessagePackFormatter] on an element type, an owner, a collection class or a member makes that value one whole value: it + // has no element ids and is off the id path. The attribute on the id member itself is the one refusal that stays. + [Test] + public void AFormatterAttributeMakesAWholeValueExceptOnTheId() + { + Assert.That(Problems(typeof(HFormattedId)).Single(), Does.Contain("Id").And.Contain("[MessagePackFormatter]")); + foreach (var (type, holder) in new (Type, object)[] + { + (typeof(HFormattedElement), new HFormattedElement { Items = { new FormattedElement() } }), + (typeof(HFormattedList), new HFormattedList { Items = { new EIded() } }), + (typeof(HFormattedOwner), new HFormattedOwner { Owner = { Items = { new EIded() } } }), + (typeof(HFormattedMember), new HFormattedMember { Inner = { Items = { new EIded() } } }), + (typeof(HFormattedDerived), new HFormattedDerived { Items = { new FormattedDerived() } }), + }) + { + Assert.That(Problems(type), Is.Empty, type.Name); + Assert.That(CultElementIds.Inspect(type).IdPath, Is.Empty, type.Name); + Assert.That(Assign(holder, "r", deterministic: false), Is.EqualTo(0), type.Name); + } + + Assert.That(Problems(typeof(HFormattedField)), Is.Empty, "a formatted type with no list below it is a field"); + } + + // The collection types between a member and its object list are written by some formatter too, so they sit on the id path. + // A value list (of a struct, a primitive or a non-keyed class) never reaches an object, so it does not. + [Test] + public void ACollectionTypeOnTheWayToAnObjectListIsOnTheIdPath() + { + var path = CultElementIds.Inspect(typeof(HCollections)).IdPath; + Assert.That(path, Is.EquivalentTo(new[] + { + typeof(HCollections), typeof(EIded), typeof(HHasIds), + typeof(List), typeof(EIded[]), typeof(Dictionary), typeof(List>) + })); + + var valueLists = CultElementIds.Inspect(typeof(HValueListsBesideObjects)).IdPath; + Assert.That(valueLists, Is.EquivalentTo(new[] { typeof(HValueListsBesideObjects), typeof(List), typeof(EIded) })); + } + + [Test] + public void AFormatterOnACollectionClassOfAnObjectListIsAWholeValue() + { + Assert.That(Problems(typeof(HRoles)), Is.Empty); + Assert.That(Assign(new HRoles { Items = { new EIded() } }, "r", deterministic: false), Is.EqualTo(0), "its elements carry no ids and are never minted"); + Assert.That(Problems(typeof(HFormattedValueCollection)), Is.Empty, "a collection class of a non-keyed element is a value list"); + } + + // MemberInfo.IsDefined ignores inherit for properties: an override sees its base's attribute only through Declares. + [Test] + public void AnInheritedFormatterOnAnOverriddenMemberIsHonoured() + { + Assert.That(Problems(typeof(HOverFormattedId)).Single(), Does.Contain("Id").And.Contain("[MessagePackFormatter]")); + Assert.That(Problems(typeof(OwnerOverFormatted)), Is.Empty, "the override inherits its base's formatter, so the member is a whole value"); + Assert.That(CultElementIds.Inspect(typeof(OwnerOverFormatted)).IdPath, Is.Empty, "a walked override would put its list on the id path"); + Assert.That(Assign(new OwnerOverFormatted { Items = { new EIded() } }, "r", deterministic: false), Is.EqualTo(0), "and mint into a list its formatter may drop ids from"); + + Assert.That(Problems(typeof(HIdOverride)), Is.Empty, "a base [CultElementId] is the override's id"); + var root = new HIdOverride { Items = { new IdOverride() } }; + Assert.That(Assign(root, "r", deterministic: false), Is.EqualTo(1)); + Assert.That(root.Items[0].Id, Has.Length.EqualTo(12)); + } + + // The union arm has a valid id, so IdProblem passes it and only the rule that an element is written by its [Key] layout + // can refuse it; it is refused once per document however many lists hold it. + [Test] + public void ANonKeyedArmWithAValidIdIsRefusedOncePerType() + { + Assert.That(Problems(typeof(HUnkeyedArmList)).Single(), Does.Contain("UnkeyedArm").And.Contain("formatter the cache does not own")); + Assert.That(Problems(typeof(HUnkeyedTwoLists)).Single(), Does.Contain("UnkeyedArm")); + Assert.That(Problems(typeof(HUnkeyedArmField)), Is.Empty); + } + + // The id path is a fixed point, not the stack at the moment a list is found: a type in a cycle with the list's owner, and + // a type reached twice, are both on the way to it. + [Test] + public void TheIdPathHoldsEveryTypeThatReachesAnObjectList() + { + var path = CultElementIds.Inspect(typeof(HDiamond)).IdPath; + Assert.That(path, Is.EquivalentTo(new[] { typeof(HDiamond), typeof(HLeft), typeof(HRight), typeof(HCycle), typeof(HCycleBack), typeof(EIded), typeof(List) })); + Assert.That(CultElementIds.Inspect(typeof(HFine)).IdPath, Is.EquivalentTo(new[] { typeof(HFine), typeof(ENest), typeof(EIded), typeof(List), typeof(List) })); + Assert.That(CultElementIds.Inspect(typeof(HFormattedField)).IdPath, Is.Empty); + + // A union base is on the path both as the type a list holds (its formatter writes the arm) and as a field type whose + // arm holds the list. + Assert.That(CultElementIds.Inspect(typeof(HPlainBaseList)).IdPath, Is.EquivalentTo(new[] { typeof(HPlainBaseList), typeof(PlainBase), typeof(PlainArm), typeof(List) })); + Assert.That(CultElementIds.Inspect(typeof(HIdBaseOwner)).IdPath, Is.EquivalentTo(new[] { typeof(HIdBaseOwner), typeof(IdBase), typeof(IdArm), typeof(EIded), typeof(List) })); + } + + // A keyed struct is a value MessagePack writes by key, and an object list inside one carries element ids like any other. + // It is walked as an owner and is never an element: a boxed copy cannot carry a minted id. + [Test] + public void AKeyedStructIsWalkedAsAnOwnerAndNeverAnElement() + { + var root = new HStructIded + { + Hold = new SHoldIded { Items = new List { new EIded() } }, + Many = { new SHoldIded { Items = new List { new EIded() } } }, + }; + Assert.That(Problems(typeof(HStructIded)), Is.Empty); + Assert.That(CultElementIds.Inspect(typeof(HStructIded)).IdPath, + Is.EquivalentTo(new[] { typeof(HStructIded), typeof(SHoldIded), typeof(List), typeof(List), typeof(EIded) })); + Assert.That(CultElementIds.Holds(typeof(HStructIded), root), Is.False); + Assert.That(Assign(root, "r", deterministic: false), Is.EqualTo(2)); + Assert.That(root.Hold.Items[0].Id, Is.Not.Empty, "visible through the struct copy"); + Assert.That(root.Many[0].Items[0].Id, Is.Not.Empty, "visible through the struct in a list"); + Assert.That(CultElementIds.Holds(typeof(HStructIded), root), Is.True); + var one = new HStructIded { Hold = new SHoldIded { Items = new List { new EIded { Id = "0123456789ab" } } } }; + Assert.That(CultElementIds.Holds(typeof(HStructIded), one), Is.True); + + Assert.That(Problems(typeof(HStructValues)), Is.Empty); + Assert.That(CultElementIds.Inspect(typeof(HStructValues)).IdPath, + Is.EquivalentTo(new[] { typeof(HStructValues), typeof(List), typeof(EIded) }), "a list of keyed structs is a value list"); + var values = new HStructValues { Structs = { new SValue { N = 1 } }, Objects = { new EIded() } }; + Assert.That(Assign(values, "r", deterministic: false), Is.EqualTo(1)); + + var nullable = new HNullableStructIded { Hold = new SHoldIded { Items = new List { new EIded() } } }; + Assert.That(Assign(nullable, "r", deterministic: false), Is.EqualTo(1), "a nullable struct is walked as its struct"); + Assert.That(CultElementIds.Holds(typeof(HNullableStructIded), nullable), Is.True); + } + + // A type is an owner or an element only when MessagePack serializes it by key ([MessagePackObject], inherited, or + // [Union]). A struct without either is a value however many [Key] members it carries. + [Test] + public void OnlyAKeyedTypeIsWalked() + { + Assert.That(Problems(typeof(HUnkeyedStruct)), Is.Empty); + Assert.That(CultElementIds.Inspect(typeof(HUnkeyedStruct)).IdPath, Is.Empty); + Assert.That(Problems(typeof(HInheritedBare)).Single(), Does.Contain("InheritedBare").And.Contain("[CultElementId]"), + "[MessagePackObject] reaches a subclass through inheritance"); + } + + // Every dictionary interface MessagePack writes as a map is walked by one path, and the minted path text is the one a + // store loaded before ids already holds, so its deterministic ids do not move. + [Test] + public void EveryMapInterfaceIsWalkedByTheSamePath() + { + var element = new Func(() => new HHasIds { Items = { new EIded() } }); + var viaInterface = new HReadOnlyIded { Map = new Dictionary { ["k"] = element() } }; + var viaClass = new HReadOnlyIded { Map = new RoMap(new Dictionary { ["k"] = element() }) }; + var viaDictionary = new HMapIded { Map = { ["k"] = element() } }; + Assert.That(Assign(viaInterface, "r", deterministic: true), Is.EqualTo(1)); + Assert.That(Assign(viaClass, "r", deterministic: true), Is.EqualTo(1), "a class implementing only IReadOnlyDictionary<,>"); + Assert.That(Assign(viaDictionary, "r", deterministic: true), Is.EqualTo(1)); + var id = viaDictionary.Map["k"].Items[0].Id; + Assert.That(viaInterface.Map["k"].Items[0].Id, Is.EqualTo(id)); + Assert.That(viaClass.Map["k"].Items[0].Id, Is.EqualTo(id)); + + // SHA-256 of "||0": slot 0, the key in braces, slot 0, the index. + var hash = System.Security.Cryptography.SHA256.HashData(System.Text.Encoding.UTF8.GetBytes("r|r.0{k}.0/0|0")); + Assert.That(id, Is.EqualTo(Convert.ToHexString(hash, 0, 6).ToLowerInvariant()), "the path text is unchanged"); + + var held = new HReadOnlyIded { Map = new RoMap(new Dictionary { ["k"] = new HHasIds { Items = { new EIded { Id = "0123456789ab" } } } }) }; + Assert.That(CultElementIds.Holds(typeof(HReadOnlyIded), held), Is.True); + Assert.That(CultElementIds.Holds(typeof(HReadOnlyIded), viaInterface), Is.True); + Assert.That(CultElementIds.Holds(typeof(HMapIded), new HMapIded { Map = { ["k"] = new HHasIds { Items = { new EIded() } } } }), Is.False); + } + + // Registration judges the declared shape, so minting and reading follow it: a member declared object is untyped on disk, + // and what a subclass adds under a base-typed member is not written, so neither is walked. + [Test] + public void TheWalkFollowsTheDeclaredShape() + { + var derived = new HBase { B = new DerC { L = { new EIded(), new EIded { Id = "0123456789ab" } } } }; + Assert.That(CultElementIds.Plan(typeof(HBase), derived, "r", deterministic: false).Count, Is.Zero); + Assert.That(CultElementIds.Holds(typeof(HBase), derived), Is.False); + var opaque = new HObject { O = new HHasIds { Items = { new EIded(), new EIded { Id = "0123456789ab" } } } }; + Assert.That(CultElementIds.Plan(typeof(HObject), opaque, "r", deterministic: false).Count, Is.Zero); + Assert.That(CultElementIds.Holds(typeof(HObject), opaque), Is.False); + foreach (var type in new[] { typeof(HBase), typeof(HObject) }) + { + Assert.That(Problems(type), Is.Empty, type.Name); + Assert.That(CultElementIds.Inspect(type).IdPath, Is.Empty, type.Name); + } + + // A union base still writes its arm, so the arm is walked. + var arm = new HIdBaseOwner { Root = new IdArm { Items = { new EIded() } } }; + Assert.That(Assign(arm, "r", deterministic: false), Is.EqualTo(1)); + } + + // MessagePack writes a non-public [Key] member only under AllowPrivate, so a non-public member of a type without it is nobody's + // id business; under AllowPrivate it is written, read and minted like any other member. + [Test] + public void ANonPublicMemberIsWalkedOnlyUnderAllowPrivate() + { + foreach (var type in new[] { typeof(HHiddenList), typeof(HHiddenOwner), typeof(HHiddenFromBase) }) + { + Assert.That(Problems(type), Is.Empty, type.Name); + Assert.That(CultElementIds.Inspect(type).IdPath, Is.Empty, type.Name); + } + + Assert.That(Problems(typeof(HOptHidden)), Is.Empty); + Assert.That(CultElementIds.Inspect(typeof(HOptHidden)).IdPath, Is.EquivalentTo(new[] { typeof(HOptHidden), typeof(List), typeof(EIded) })); + var opt = new HOptHidden(); + opt.Peek.Add(new EIded()); + opt.Peek.Add(new EIded()); + Assert.That(Assign(opt, "r", deterministic: false), Is.EqualTo(2)); + Assert.That(opt.Peek.Select(item => item.Id), Has.All.Match("^[0-9a-f]{12}$")); + + Assert.That(Problems(typeof(HHiddenLeaf)), Is.Empty); + Assert.That(CultElementIds.Inspect(typeof(HHiddenLeaf)).IdPath, Is.EquivalentTo(new[] { typeof(HHiddenLeaf), typeof(List), typeof(EIded) }), + "a non-public member adds nothing to the id path"); + } + + // MessagePack writes a [MessagePackObject] class that is only IEnumerable by its keys, so the walk reads it by key. A + // keyed tree node that enumerates its own children is the shape that overflowed the stack when enumeration came first. + [Test] + public void ATypeThatEnumeratesItselfIsWalkedByKeyAndTerminates() + { + var survey = CultElementIds.Inspect(typeof(HKNode)); + Assert.That(survey.Problems, Is.Empty); + Assert.That(survey.IdPath, Is.EquivalentTo(new[] { typeof(HKNode), typeof(List), typeof(KNode) })); + + var root = new HKNode { Nodes = { new KNode { Children = { new KNode { Children = { new KNode() } } } } } }; + Assert.That(Assign(root, "r", deterministic: true), Is.EqualTo(3)); + Assert.That(root.Nodes[0].Id, Has.Length.EqualTo(12)); + Assert.That(root.Nodes[0].Children[0].Children[0].Id, Has.Length.EqualTo(12)); + var options = GameCult.Caching.MessagePack.CultDocumentMessagePackSerialization.Options; + var back = MessagePackSerializer.Deserialize(MessagePackSerializer.Serialize(root, options), options); + Assert.That(back.Nodes[0].Id, Is.EqualTo(root.Nodes[0].Id), "MessagePack writes the node by key, ids included"); + Assert.That(back.Nodes[0].Children[0].Children[0].Id, Is.EqualTo(root.Nodes[0].Children[0].Children[0].Id)); + + // An unkeyed type that enumerates itself holds no element list and ends the type walk. + var unkeyed = CultElementIds.Inspect(typeof(HUNode)); + Assert.That(unkeyed.Problems, Is.Empty); + Assert.That(unkeyed.IdPath, Is.Empty); + + // A keyed class that only enumerates is read by its keyed lists, so both are judged and minted. + Assert.That(Problems(typeof(KBag)).Single(), Does.Contain("EBare")); + var bag = new KBagIded { Items = { new EIded(), new EIded() } }; + Assert.That(Assign(bag, "r", deterministic: false), Is.EqualTo(2)); + Assert.That(CultElementIds.Holds(typeof(KBagIded), bag), Is.True); + } + + // The walk reads the members MessagePack writes: without AllowPrivate a public field or a property with a public getter, with + // it every member up the base chain, the most derived [MessagePackObject] declaration deciding; no [IgnoreMember]. + [Test] + public void TheWalkReadsTheMembersMessagePackWrites() + { + Assert.That(Problems(typeof(PubIgn)), Is.Empty, "an ignored member is never written, so its list is nobody's business"); + Assert.That(Problems(typeof(ApIgn)), Is.Empty, "an ignored private member is not written"); + Assert.That(Problems(typeof(PgHold)), Is.Empty, "a list behind a private getter is not written without AllowPrivate"); + + foreach (var type in new[] { typeof(EIded), typeof(HHiddenLeaf), typeof(PubIgn), typeof(ApIgn), typeof(PgHold), typeof(MemberMix), typeof(HOptHidden), typeof(ApDerived), typeof(NoApDerived), typeof(ApInheritor), typeof(ApIgnAp), typeof(PSkH), typeof(ISkH) }) + { + var allowPrivate = type.GetCustomAttribute(true)!.AllowPrivate; + Assert.That(CultElementIds.WrittenKeyedMembers(type).Select(member => member.Name), + Is.EquivalentTo(MessagePackMembers(type, allowPrivate)), type.Name); + } + + Assert.That(CultElementIds.WrittenKeyedMembers(typeof(NoApDerived)).Select(member => member.Name), Is.EquivalentTo(new[] { "N" }), + "the derived declaration decides, so an AllowPrivate base adds nothing"); + } + + // The names of the members MessagePack's own discovery reads for a type, by reflection into its internal type: the + // oracle the walk's member set is pinned to. + private static List MessagePackMembers(Type type, bool allowPrivate) + { + const string moved = "MessagePack moved its member discovery; re-derive MembersOf"; + const System.Reflection.BindingFlags all = System.Reflection.BindingFlags.Static | System.Reflection.BindingFlags.Instance | + System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.NonPublic; + var info = typeof(MessagePackSerializer).Assembly.GetType("MessagePack.Internal.ObjectSerializationInfo"); + var create = info?.GetMethod("CreateOrNull", all); + if (create == null || create.GetParameters().Length != 4) + { + Assert.Fail(moved); + return new List(); + } + + var result = create.Invoke(null, new object[] { type, false, false, allowPrivate }); + var members = result?.GetType().GetProperty("Members", all)?.GetValue(result) as System.Collections.IEnumerable; + if (members == null) + { + Assert.Fail(moved); + return new List(); + } + + var names = new List(); + foreach (var member in members) + { + var readable = member.GetType().GetProperty("IsReadable", all)?.GetValue(member); + var name = member.GetType().GetProperty("Name", all)?.GetValue(member) as string; + if (readable is not bool || name == null) + Assert.Fail(moved); + if ((bool)readable!) + names.Add(name!); + } + + return names; + } + + // An id MessagePack writes but cannot read back would be re-minted differently on every load; one it does not write + // never reaches disk. + [Test] + public void AnIdMessagePackCannotReadBackIsRefused() + { + Assert.That(Problems(typeof(HPrivSet)).Single(), Does.Contain("setter is not public").And.Contain("Id")); + Assert.That(Problems(typeof(HIgnId)).Single(), Does.Contain("does not write").And.Contain("Id")); + Assert.That(Problems(typeof(HPrivateId)).Single(), Does.Contain("does not write").And.Contain("id")); + Assert.That(Problems(typeof(HInit)), Is.Empty); + Assert.That(Problems(typeof(HCtorId)), Is.Empty); + + foreach (var holder in new object[] + { + new HInit { Items = { new EInit(), new EInit() } }, + new HCtorId { Items = { new ECtorId(null), new ECtorId(null) } }, + }) + { + Assert.That(Assign(holder, "r", deterministic: false), Is.EqualTo(2), holder.GetType().Name); + var bytes = MessagePackSerializer.Serialize(holder.GetType(), holder, GameCult.Caching.MessagePack.CultDocumentMessagePackSerialization.Options); + var back = MessagePackSerializer.Deserialize(holder.GetType(), bytes, GameCult.Caching.MessagePack.CultDocumentMessagePackSerialization.Options); + Assert.That(Ids(back), Is.EqualTo(Ids(holder)), holder.GetType().Name); + Assert.That(Ids(back), Has.All.Match("^[0-9a-f]{12}$"), holder.GetType().Name); + } + } + + // Every shape the walk does not own is one whole value: no problem, off the id path, never minted into and never counted. + [Test] + public void AShapeTheWalkDoesNotOwnIsOneWholeValue() + { + const string Set = "aaaaaaaaaaaa"; + EIded E(string id) => new() { Id = id }; + var holders = new (Type Type, Func Make)[] + { + (typeof(Whole>), id => new Whole> { Value = ImmutableArray.Create(E(id)) }), + (typeof(Whole?>), id => new Whole?> { Value = ImmutableArray.Create(E(id)) }), + (typeof(Whole>), id => new Whole> { Value = new ArraySegment(new[] { E(id) }) }), + (typeof(Whole>), id => new Whole> { Value = new[] { E(id) } }), + (typeof(Whole>), id => new Whole> { Value = new[] { E(id) } }), + (typeof(Whole>>), id => new Whole>> { Value = new("k", new List { E(id) }) }), + (typeof(Whole<(List, int)>), id => new Whole<(List, int)> { Value = (new List { E(id) }, 1) }), + (typeof(Whole, int>>), id => new Whole, int>> { Value = Tuple.Create(new List { E(id) }, 1) }), + (typeof(Whole>>), id => new Whole>> { Value = new Lazy>(() => new List { E(id) }) }), + (typeof(Whole), id => new Whole { Value = new NkHold { Items = { E(id) } } }), + (typeof(Whole), id => new Whole { Value = new SkHold { Items = { E(id) } } }), + (typeof(Whole), id => new Whole { Value = new DcHold { Items = { E(id) } } }), + (typeof(Whole), id => new Whole { Value = new KpBase { Items = { E(id) } } }), + (typeof(Whole), id => new Whole { Value = new KpDerived { Items = { E(id) } } }), + (typeof(HRoles), id => new HRoles { Items = { E(id) } }), + (typeof(HFormattedList), id => new HFormattedList { Items = { E(id) } }), + (typeof(HHiddenList), id => new HHiddenList()), + }; + foreach (var (type, make) in holders) + { + Assert.That(Problems(type), Is.Empty, type.ToString()); + Assert.That(CultElementIds.Inspect(type).IdPath, Is.Empty, type.ToString()); + Assert.That(CultElementIds.CanHoldIds(type), Is.False, type.ToString()); + Assert.That(Assign(make(string.Empty), "r", deterministic: false), Is.EqualTo(0), type.ToString()); + Assert.That(Assign(make(string.Empty), "r", deterministic: true), Is.EqualTo(0), type.ToString()); + Assert.That(CultElementIds.Holds(type, make(Set)), Is.False, type.ToString()); + } + + // A [DataContract] subclass of a [MessagePackObject] type is still read by key: the attribute it inherits keeps it an owner. + Assert.That(CultElementIds.Inspect(typeof(Whole)).IdPath, Does.Contain(typeof(List))); + + // Elements of a string-keyed type carry no id either, and an owned list of a bare element is still refused. + Assert.That(Problems(typeof(Whole>)), Is.Empty); + RefusesBare(typeof(Whole>)); + } + + // A string key counts only on a member MessagePack writes: a private or [IgnoreMember] one leaves the type integer-keyed and owned. + [Test] + public void AStringKeyOnAMemberMessagePackDoesNotWriteLeavesTheTypeOwned() + { + foreach (var holder in new object[] { new PSkH { Items = { new EIded(), new EIded() } }, new ISkH { Items = { new EIded(), new EIded() } } }) + { + Assert.That(Problems(holder.GetType()), Is.Empty, holder.GetType().Name); + Assert.That(CultElementIds.Inspect(holder.GetType()).IdPath, Does.Contain(holder.GetType()), holder.GetType().Name); + Assert.That(Assign(holder, "r", deterministic: false), Is.EqualTo(2), holder.GetType().Name); + Assert.That(CultElementIds.Holds(holder.GetType(), holder), Is.True, holder.GetType().Name); + } + + Assert.That(CultElementIds.WalksAsOwner(typeof(SkOver)), Is.False, "a string key inherited by an override still makes the type one whole value"); + Assert.That(CultElementIds.WalksAsOwner(typeof(PSkH)), Is.True); + Assert.That(Problems(typeof(Whole>)).Single(), Does.Contain("[CultElementId]")); + Assert.That(Problems(typeof(Whole>)).Single(), Does.Contain("[CultElementId]"), "an element whose only string key is ignored is as bare as its twin"); + } + + // The AllowPrivate edges: an [IgnoreMember] member is not written even under it, and only the element's own most derived + // declaration lets a private setter be read back. + [Test] + public void TheAllowPrivateEdgesFollowWhatMessagePackWrites() + { + Assert.That(CultElementIds.WrittenKeyedMembers(typeof(ApIgnAp)), Is.Empty, "an ignored member is not written under AllowPrivate"); + Assert.That(Problems(typeof(HApIgnId)).Single(), Does.Contain("does not write").And.Contain("Id")); + Assert.That(Problems(typeof(HNoApPrivSet)).Single(), Does.Contain("setter is not public").And.Contain("Id"), + "the derived plain [MessagePackObject] decides: its private setter is not read back, whatever its base declares"); + } + + // MessagePack writes an AllowPrivate type with every member up its base chain, and the most derived [MessagePackObject] + // declaration decides whether that is so. + [Test] + public void AnAllowPrivateTypeIsWalkedByTheMembersMessagePackWrites() + { + Assert.That(CultElementIds.WrittenKeyedMembers(typeof(ApDerived)).Select(member => member.Name), Is.EquivalentTo(new[] { "baseStash", "own", "N", "Pg" })); + Assert.That(CultElementIds.WrittenKeyedMembers(typeof(NoApDerived)).Select(member => member.Name), Is.EquivalentTo(new[] { "N" })); + Assert.That(CultElementIds.WrittenKeyedMembers(typeof(ApInheritor)).Select(member => member.Name), Is.EquivalentTo(new[] { "baseStash", "own", "N", "Pg" }), + "AllowPrivate is inherited by a subclass that declares no attribute of its own"); + + var options = GameCult.Caching.MessagePack.CultDocumentMessagePackSerialization.Options; + var held = new HOptHidden(); + held.Peek.Add(new EIded()); + Assert.That(Assign(held, "r", deterministic: false), Is.EqualTo(1)); + var back = MessagePackSerializer.Deserialize(MessagePackSerializer.Serialize(held, options), options); + Assert.That(back.Peek.Single().Id, Is.EqualTo(held.Peek.Single().Id), "the minted id of a private list survives a round trip"); + + Assert.That(Problems(typeof(HApPrivSet)), Is.Empty, "MessagePack reads an AllowPrivate type's private setter back"); + var privSet = new HApPrivSet { Items = { new EApPrivSet(), new EApPrivSet() } }; + Assert.That(Assign(privSet, "r", deterministic: false), Is.EqualTo(2)); + var round = MessagePackSerializer.Deserialize(MessagePackSerializer.Serialize(privSet, options), options); + Assert.That(round.Items.Select(item => item.Id), Is.EqualTo(privSet.Items.Select(item => item.Id))); + Assert.That(round.Items.Select(item => item.Id), Has.All.Match("^[0-9a-f]{12}$")); + Assert.That(Problems(typeof(HPrivSet)).Single(), Does.Contain("setter is not public"), "without AllowPrivate it is still refused"); + } + + private static string?[] Ids(object holder) => holder switch + { + HInit init => init.Items.Select(item => item.Id).ToArray(), + HCtorId ctor => ctor.Items.Select(item => item.Id).ToArray(), + _ => throw new ArgumentException(holder.GetType().Name), + }; + + // A subclass that redeclares the id member does not change which member the declared element writes. + [Test] + public void AnElementIsReadByItsDeclaredTypeNotItsRuntimeType() + { + Assert.That(Problems(typeof(HDoc2)), Is.Empty); + static HE Hd(string? declared = null, string? redeclared = null) + { + var item = new HD { Id = redeclared }; + ((HE)item).Id = declared; + return item; + } + + var minted = new HDoc2 { Items = { Hd() } }; + Assert.That(Assign(minted, "r", deterministic: false), Is.EqualTo(1)); + Assert.That(minted.Items[0].Id, Does.Match("^[0-9a-f]{12}$")); + Assert.That(((HD)minted.Items[0]).Id, Is.Null, "the redeclared member is never written, so it is not minted"); + + Assert.That(() => CultElementIds.Plan(typeof(HDoc2), new HDoc2 { Items = { Hd(declared: "not-hex") } }, "r", false), + Throws.TypeOf(), "the declared member is checked"); + Assert.That(() => CultElementIds.Plan(typeof(HDoc2), new HDoc2 { Items = { Hd(redeclared: "not-hex") } }, "r", false), + Throws.Nothing, "the redeclared member is never read"); + + Assert.That(CultElementIds.Holds(typeof(HDoc2), new HDoc2 { Items = { Hd(declared: "0123456789ab") } }), Is.True); + Assert.That(CultElementIds.Holds(typeof(HDoc2), new HDoc2 { Items = { Hd(redeclared: "0123456789ab") } }), Is.False); + } + + // ---- shapes ---- + [MessagePackObject] public class PubIgn { [Key(0)] [IgnoreMember] public List L { get; set; } = new(); } +#pragma warning disable CS0169, CS0414, CS0649 + [MessagePackObject] public class ApIgn { [Key(0)] [IgnoreMember] private List l = new(); } +#pragma warning restore CS0169, CS0414, CS0649 + [MessagePackObject] public class PgHold { [Key(0)] public List Items { private get; set; } = new(); } + [MessagePackObject] + public class MemberMix + { + [Key(0)] public int A; + [Key(1)] [IgnoreMember] public List? Ign; + [Key(2)] public string? PS { get; private set; } + [Key(3)] public string? GetOnly { get; } +#pragma warning disable CS0169, CS0414, CS0649 + [Key(4)] private int priv; +#pragma warning restore CS0169, CS0414, CS0649 + [Key(5)] public string? InitOnly { get; init; } + [IgnoreMember] public int Priv => priv; + } + + [MessagePackObject] public class EPrivSet { [Key(0)] [CultElementId] public string? Id { get; private set; } } + [MessagePackObject] public class HPrivSet { [Key(0)] public List Items { get; set; } = new(); } + [MessagePackObject] public class EIgnId { [Key(0)] [IgnoreMember] [CultElementId] public string? Id { get; set; } } + [MessagePackObject] public class HIgnId { [Key(0)] public List Items { get; set; } = new(); } + #pragma warning disable CS0169, CS0649 + [MessagePackObject] public class EPrivateId { [Key(0)] [CultElementId] private string? id; } +#pragma warning restore CS0169, CS0649 + [MessagePackObject] public class HPrivateId { [Key(0)] public List Items { get; set; } = new(); } + [MessagePackObject] public class EInit { [Key(0)] [CultElementId] public string? Id { get; init; } } + [MessagePackObject] public class HInit { [Key(0)] public List Items { get; set; } = new(); } + [MessagePackObject] + public class ECtorId + { + [SerializationConstructor] public ECtorId(string? id) { Id = id; } + [Key(0)] [CultElementId] public string? Id { get; set; } + } + [MessagePackObject] public class HCtorId { [Key(0)] public List Items { get; set; } = new(); } + + [MessagePackObject] public class HE { [Key(0)] [CultElementId] public string? Id { get; set; } } + [MessagePackObject] public class HD : HE { [Key(2)] [CultElementId] public new string? Id { get; set; } } + [MessagePackObject] public class HDoc2 { [Key(0)] public List Items { get; set; } = new(); } + + [MessagePackObject] + public class KNode : IEnumerable + { + [Key(0)] [CultElementId] public string? Id { get; set; } + [Key(1)] public List Children { get; set; } = new(); + public IEnumerator GetEnumerator() => Children.GetEnumerator(); + System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => GetEnumerator(); + } + [MessagePackObject] public class HKNode { [Key(0)] public List Nodes { get; set; } = new(); } + public class UNode : IEnumerable + { + public IEnumerator GetEnumerator() { yield break; } + System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => GetEnumerator(); + } + [MessagePackObject] public class HUNode { [Key(0)] public UNode? N { get; set; } } + [MessagePackObject] + public class KBag : IEnumerable + { + [Key(0)] public List Items { get; set; } = new(); + [Key(1)] public List Other { get; set; } = new(); + public IEnumerator GetEnumerator() { yield break; } + System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => GetEnumerator(); + } + [MessagePackObject] + public class KBagIded : IEnumerable + { + [Key(0)] public List Items { get; set; } = new(); + public IEnumerator GetEnumerator() { yield break; } + System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => GetEnumerator(); + } + + [MessagePackObject] public class EBare { [Key(0)] public string X { get; set; } = ""; } + [MessagePackObject] public class EIded { [Key(0)] public string X { get; set; } = ""; [Key(1)] [CultElementId] public string Id { get; set; } = ""; } + [MessagePackObject] public class ENest { [Key(0)] [CultElementId] public string Id { get; set; } = ""; [Key(1)] public List Sub { get; set; } = new(); } + [MessagePackObject] public class EBareNest { [Key(0)] [CultElementId] public string Id { get; set; } = ""; [Key(1)] public List Sub { get; set; } = new(); } + + public sealed class NoFormatter { } + public class UnkeyedArm : ArmBase { [Key(0)] [CultElementId] public string Id { get; set; } = ""; } + [Union(0, typeof(KeyedArm)), Union(1, typeof(UnkeyedArm))] public abstract class ArmBase { } + [MessagePackObject] public class KeyedArm : ArmBase { [Key(0)] [CultElementId] public string Id { get; set; } = ""; } + [MessagePackObject] public class HUnkeyedArmList { [Key(0)] public List Items { get; set; } = new(); } + [MessagePackObject] public class HUnkeyedArmField { [Key(0)] public ArmBase? Root { get; set; } } + [MessagePackObject] + public class HCollections + { + [Key(0)] public List A { get; set; } = new(); + [Key(1)] public EIded[] B { get; set; } = Array.Empty(); + [Key(2)] public Dictionary C { get; set; } = new(); + [Key(3)] public List> D { get; set; } = new(); + } + + [MessagePackObject] + public class HValueListsBesideObjects + { + [Key(0)] public List Objects { get; set; } = new(); + [Key(1)] public List Numbers { get; set; } = new(); + [Key(2)] public List Plain { get; set; } = new(); + [Key(3)] public List Points { get; set; } = new(); + } + + [MessagePackFormatter(typeof(NoFormatter))] public class Roles : List { } + [MessagePackObject] public class HRoles { [Key(0)] public Roles Items { get; set; } = new(); } + [MessagePackFormatter(typeof(NoFormatter))] public class PlainValues : List { } + [MessagePackObject] public class HFormattedValueCollection { [Key(0)] public List Objects { get; set; } = new(); [Key(1)] public PlainValues Plain { get; set; } = new(); } + + [MessagePackObject] + public abstract class FormattedIdBase + { + [Key(0)] [MessagePackFormatter(typeof(NoFormatter))] public virtual string Id { get; set; } = ""; + } + + [MessagePackObject] + public class FormattedIdOverride : FormattedIdBase + { + [Key(0)] [CultElementId] public override string Id { get; set; } = ""; + } + + [MessagePackObject] public class HOverFormattedId { [Key(0)] public List Items { get; set; } = new(); } + + [MessagePackObject] + public class OwnerBaseFormatted + { + [Key(0)] [MessagePackFormatter(typeof(NoFormatter))] public virtual List Items { get; set; } = new(); + } + + [MessagePackObject] + public class OwnerOverFormatted : OwnerBaseFormatted + { + [Key(0)] public override List Items { get; set; } = new(); + } + + [MessagePackObject] + public class IdVirtual + { + [Key(0)] public string X { get; set; } = ""; + [Key(1)] [CultElementId] public virtual string Id { get; set; } = ""; + } + + [MessagePackObject] public class IdOverride : IdVirtual { [Key(1)] public override string Id { get; set; } = ""; } + [MessagePackObject] public class HIdOverride { [Key(0)] public List Items { get; set; } = new(); } + + [MessagePackObject] public class HUnkeyedTwoLists { [Key(0)] public List A { get; set; } = new(); [Key(1)] public List B { get; set; } = new(); } + public class PlainValue { public string X { get; set; } = ""; } + [MessagePackObject] public class HValueList { [Key(0)] public List Items { get; set; } = new(); } + + [MessagePackObject] [MessagePackFormatter(typeof(NoFormatter))] public class FormattedElement { [Key(0)] [CultElementId] public string Id { get; set; } = ""; } + [MessagePackObject] public class HFormattedElement { [Key(0)] public List Items { get; set; } = new(); } + [MessagePackObject] public class HFormattedList { [Key(0)] [MessagePackFormatter(typeof(NoFormatter))] public List Items { get; set; } = new(); } + [MessagePackObject] public class FormattedIdElement { [Key(0)] [CultElementId] [MessagePackFormatter(typeof(NoFormatter))] public string Id { get; set; } = ""; } + [MessagePackObject] public class HFormattedId { [Key(0)] public List Items { get; set; } = new(); } + [MessagePackObject] [MessagePackFormatter(typeof(NoFormatter))] public class FormattedOwner { [Key(0)] public List Items { get; set; } = new(); } + [MessagePackObject] public class HFormattedOwner { [Key(0)] public FormattedOwner Owner { get; set; } = new(); } + [MessagePackObject] public class HFormattedMember { [Key(0)] [MessagePackFormatter(typeof(NoFormatter))] public HHasIds Inner { get; set; } = new(); } + [MessagePackObject] [MessagePackFormatter(typeof(NoFormatter))] public class FormattedLeaf { [Key(0)] public int N { get; set; } } + [MessagePackObject] public class HFormattedField { [Key(0)] public FormattedLeaf Leaf { get; set; } = new(); [Key(1)] public List Tags { get; set; } = new(); } + + [MessagePackObject] public class FormattedDerived : FormattedElement { } + [MessagePackObject] public class HFormattedDerived { [Key(0)] public List Items { get; set; } = new(); } + [Union(0, typeof(IdArm))] public abstract class IdBase { } + [MessagePackObject] public class IdArm : IdBase { [Key(0)] [CultElementId] public string Id { get; set; } = ""; [Key(1)] public List Items { get; set; } = new(); } + [Union(0, typeof(PlainArm))] public abstract class PlainBase { } + [MessagePackObject] public class PlainArm : PlainBase { [Key(0)] [CultElementId] public string Id { get; set; } = ""; } + [MessagePackObject] public class HPlainBaseList { [Key(0)] public List Items { get; set; } = new(); } + [MessagePackObject] public class HIdBaseOwner { [Key(0)] public IdBase? Root { get; set; } } + + [MessagePackObject] public class HLeft { [Key(0)] public HCycle Cycle { get; set; } = new(); } + [MessagePackObject] public class HRight { [Key(0)] public HCycle Cycle { get; set; } = new(); } + [MessagePackObject] public class HCycle { [Key(0)] public HCycleBack Back { get; set; } = new(); } + [MessagePackObject] public class HCycleBack { [Key(0)] public HCycle? Up { get; set; } [Key(1)] public List Items { get; set; } = new(); } + [MessagePackObject] public class HDiamond { [Key(0)] public HLeft Left { get; set; } = new(); [Key(1)] public HRight Right { get; set; } = new(); } + + [MessagePackObject] public class HList { [Key(0)] public List Items { get; set; } = new(); } + [MessagePackObject] public class HArray { [Key(0)] public EBare[] Items { get; set; } = Array.Empty(); } + [MessagePackObject] public class HDictionary { [Key(0)] public Dictionary Map { get; set; } = new(); } + [MessagePackObject] public class HNested { [Key(0)] public HList Inner { get; set; } = new(); } + [MessagePackObject] public class HNullableInner { [Key(0)] public HList? Inner { get; set; } } + [MessagePackObject] public class HListOfLists { [Key(0)] public List> Grid { get; set; } = new(); } + [MessagePackObject] public class HElementNest { [Key(0)] public List Items { get; set; } = new(); } + [MessagePackObject] public class HFine { [Key(0)] public List Items { get; set; } = new(); [Key(1)] public List Tags { get; set; } = new(); [Key(2)] public int[] Numbers { get; set; } = Array.Empty(); } + + [MessagePackObject] + [Union(0, typeof(UnionSub))] + public abstract class UnionBase { [Key(0)] [CultElementId] public string Id { get; set; } = ""; } + [MessagePackObject] public class UnionSub : UnionBase { [Key(1)] public List Hidden { get; set; } = new(); } + [MessagePackObject] public class HUnion { [Key(0)] public List Items { get; set; } = new(); } + + [MessagePackObject] public class ETwo { [Key(0)] [CultElementId] public string A { get; set; } = ""; [Key(1)] [CultElementId] public string B { get; set; } = ""; } + [MessagePackObject] public class HTwo { [Key(0)] public List Items { get; set; } = new(); } + [MessagePackObject] public class EInt { [Key(0)] [CultElementId] public int Id { get; set; } } + [MessagePackObject] public class HInt { [Key(0)] public List Items { get; set; } = new(); } + [MessagePackObject] public class ENoKey { [IgnoreMember] [CultElementId] public string Id { get; set; } = ""; } + [MessagePackObject] public class HNoKey { [Key(0)] public List Items { get; set; } = new(); } + [MessagePackObject] public class EGetOnly { [Key(0)] [CultElementId] public string Id { get; } = ""; } + [MessagePackObject] public class HGetOnly { [Key(0)] public List Items { get; set; } = new(); } + [MessagePackObject] public class EMissingSource { [Key(0)] [CultElementId("Nope")] public string Id { get; set; } = ""; } + [MessagePackObject] public class HMissingSource { [Key(0)] public List Items { get; set; } = new(); } + [MessagePackObject] public class EBadSourceKind { [Key(0)] public List Tags { get; set; } = new(); [Key(1)] [CultElementId(nameof(Tags))] public string Id { get; set; } = ""; } + [MessagePackObject] public class HBadSourceKind { [Key(0)] public List Items { get; set; } = new(); } + [MessagePackObject] public class EUnkeyedSource { [IgnoreMember] public string Loose { get; set; } = ""; [Key(0)] [CultElementId(nameof(Loose))] public string Id { get; set; } = ""; } + [MessagePackObject] public class HUnkeyedSource { [Key(0)] public List Items { get; set; } = new(); } + [MessagePackObject] public class ENullableSource { [Key(0)] public int? Number { get; set; } [Key(1)] [CultElementId(nameof(Number))] public string Id { get; set; } = ""; } + [MessagePackObject] public class HNullableSource { [Key(0)] public List Items { get; set; } = new(); } + + [MessagePackObject] public class HHasIds { [Key(0)] public List Items { get; set; } = new(); } + [MessagePackObject] + public class HAll + { + [Key(0)] public Dictionary Dict { get; set; } = new(); + [Key(1)] public List> Grid { get; set; } = new(); + [Key(2)] public List Nest { get; set; } = new(); + } + + [MessagePackObject] public struct SHold { [Key(0)] public List Items; } + [MessagePackObject] public struct SHoldIded { [Key(0)] public List Items; } + [MessagePackObject] public struct SValue { [Key(0)] public int N; } + public struct UnkeyedS { [Key(0)] public List Items; } + [MessagePackObject] public class HUnkeyedStruct { [Key(0)] public UnkeyedS Value; } + public class InheritedBare : EBare { } + [MessagePackObject] public class HInheritedBare { [Key(0)] public List Items { get; set; } = new(); } + [MessagePackObject] public class HNullableStructIded { [Key(0)] public SHoldIded? Hold; } + [MessagePackObject] public class HStruct { [Key(0)] public SHold Hold; } + [MessagePackObject] public class HStructList { [Key(0)] public List Holds { get; set; } = new(); } + [MessagePackObject] public class HNullableStruct { [Key(0)] public SHold? Hold; } + [MessagePackObject] public class HStructIded { [Key(0)] public SHoldIded Hold; [Key(1)] public List Many = new(); } + [MessagePackObject] public class HStructValues { [Key(0)] public List Structs = new(); [Key(1)] public List Objects = new(); } + + [MessagePackObject] public class HReadOnlyMap { [Key(0)] public IReadOnlyDictionary Map { get; set; } = new Dictionary(); } + [MessagePackObject] public class HReadOnlyIded { [Key(0)] public IReadOnlyDictionary Map { get; set; } = new Dictionary(); } + [MessagePackObject] public class HMapIded { [Key(0)] public Dictionary Map { get; set; } = new(); } + public sealed class RoMap : IReadOnlyDictionary + { + private readonly Dictionary _inner; + public RoMap(Dictionary inner) => _inner = inner; + public HHasIds this[string key] => _inner[key]; + public IEnumerable Keys => _inner.Keys; + public IEnumerable Values => _inner.Values; + public int Count => _inner.Count; + public bool ContainsKey(string key) => _inner.ContainsKey(key); + public bool TryGetValue(string key, out HHasIds value) => _inner.TryGetValue(key, out value!); + public IEnumerator> GetEnumerator() => _inner.GetEnumerator(); + System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => GetEnumerator(); + } + + [MessagePackObject] public class BaseC { [Key(0)] public int X { get; set; } } + [MessagePackObject] public class DerC : BaseC { [Key(1)] public List L { get; set; } = new(); } + [MessagePackObject] public class HBase { [Key(0)] public BaseC B { get; set; } = new(); } + [MessagePackObject] public class HObject { [Key(0)] public object? O { get; set; } } + +#pragma warning disable CS0169, CS0414, CS0649 + [MessagePackObject] public class HHiddenList { [Key(0)] private List Stash { get; set; } = new(); } + [MessagePackObject] public class HHiddenOwner { [Key(0)] private HHasIds stash = new(); } + [MessagePackObject(AllowPrivate = true)] public class HOptHidden { [Key(0)] private List stash = new(); [IgnoreMember] public List Peek => stash; } + [MessagePackObject] public class HHiddenBase { [Key(0)] private List stash = new(); } + [MessagePackObject] public class HHiddenFromBase : HHiddenBase { [Key(1)] public int N { get; set; } } + [MessagePackObject] public class HHiddenLeaf { [Key(0)] public List Items { get; set; } = new(); [Key(1)] private int secret = 7; } + [MessagePackObject] public class ApBase { [Key(0)] private List baseStash = new(); } + [MessagePackObject(AllowPrivate = true)] public class ApDerived : ApBase { [Key(1)] private List own = new(); [Key(2)] public int N { get; set; } [Key(3)] public List Pg { private get; set; } = new(); } + [MessagePackObject(AllowPrivate = true)] public class ApBase2 { [Key(0)] private List baseStash = new(); } + [MessagePackObject] public class NoApDerived : ApBase2 { [Key(1)] public int N { get; set; } } +#pragma warning restore CS0169, CS0414, CS0649 + + [MessagePackObject] public class Whole { [Key(0)] public T? Value { get; set; } } + public class NkHold { public List Items = new(); } + [MessagePackObject] public class SkHold { [Key("items")] public List Items = new(); } + [MessagePackObject(keyAsPropertyName: true)] public class KpBase { [Key(0)] public List Items = new(); } + public class KpDerived : KpBase { } + [MessagePackObject] public class MpBase { [Key(0)] public List Items = new(); } + [System.Runtime.Serialization.DataContract] public class DcOverMp : MpBase { } + public class ApInheritor : ApDerived { } + [MessagePackObject] public class SkElem { [Key("x")] public string X { get; set; } = ""; } + [System.Runtime.Serialization.DataContract] public class DcHold { [System.Runtime.Serialization.DataMember(Order = 0)] public List Items = new(); } + [MessagePackObject(AllowPrivate = true)] public class ApIgnAp { [Key(0)] [IgnoreMember] private List l = new(); } + [MessagePackObject] public class PSkH { [Key(0)] public List Items = new(); [Key("x")] private int x = 1; } + [MessagePackObject] public class ISkH { [Key(0)] public List Items = new(); [IgnoreMember] [Key("x")] public int X { get; set; } } + [MessagePackObject] public class SkOverBase { [Key("items")] public virtual List Items { get; set; } = new(); } + [MessagePackObject] public class SkOver : SkOverBase { public override List Items { get; set; } = new(); } + [MessagePackObject] public class NoIdEl { [Key(0)] public int N { get; set; } [IgnoreMember] [Key("x")] public int X { get; set; } } + [MessagePackObject] public class NoIdPlain { [Key(0)] public int N { get; set; } } + [MessagePackObject(AllowPrivate = true)] public class EApIgnId { [Key(0)] public int N { get; set; } [Key(1)] [IgnoreMember] [CultElementId] public string Id { get; set; } = ""; } + [MessagePackObject] public class HApIgnId { [Key(0)] public List Items { get; set; } = new(); } + [MessagePackObject(AllowPrivate = true)] public class ApIdBase { [Key(0)] public int N { get; set; } } + [MessagePackObject] public class ENoApPrivSet : ApIdBase { [Key(1)] [CultElementId] public string? Id { get; private set; } } + [MessagePackObject] public class HNoApPrivSet { [Key(0)] public List Items { get; set; } = new(); } + [MessagePackObject(AllowPrivate = true)] public class EApPrivSet { [Key(0)] [CultElementId] public string? Id { get; private set; } } + [MessagePackObject] public class HApPrivSet { [Key(0)] public List Items { get; set; } = new(); } + + [MessagePackObject] public class EDerivedText { [Key(0)] public string? Name { get; set; } [Key(1)] [CultElementId(nameof(Name))] public string Id { get; set; } = ""; } + [MessagePackObject] public class HDerivedText { [Key(0)] public List Items { get; set; } = new(); } + + [MessagePackObject] + public class HCulture + { + [Key(0)] public Dictionary ByDouble { get; set; } = new(); + [Key(1)] public Dictionary ByInt { get; set; } = new(); + [Key(2)] public Dictionary ByDate { get; set; } = new(); + } + + // An interface union: the interface declares the id, the concrete type does not, so nothing would ever mint into it. + [Union(0, typeof(IfaceBad))] + public interface IUnionIface { [Key(0)] [CultElementId] string Id { get; set; } } + [MessagePackObject] public class IfaceBad : IUnionIface { [Key(0)] public string Id { get; set; } = ""; } + [MessagePackObject] public class HIfaceBad { [Key(0)] public List Items { get; set; } = new(); } + + [Union(0, typeof(IfaceGood))] + public interface IUnionIfaceGood { } + [MessagePackObject] public class IfaceGood : IUnionIfaceGood { [Key(0)] [CultElementId] public string Id { get; set; } = ""; } + [MessagePackObject] public class HIfaceGood { [Key(0)] public List Items { get; set; } = new(); } + + // A subtype that reuses the base id's key. + [MessagePackObject] + [Union(0, typeof(KeyClash))] + public abstract class KeyClashBase { [Key(0)] [CultElementId] public string Id { get; set; } = ""; } + [MessagePackObject] public class KeyClash : KeyClashBase { [Key(0)] public string Clash { get; set; } = ""; } + [MessagePackObject] public class HKeyClash { [Key(0)] public List Items { get; set; } = new(); } + + [MessagePackObject] + [Union(0, typeof(DupSub))] + [Union(1, typeof(DupSub))] + public abstract class DupBase { } + [MessagePackObject] public class DupSub : DupBase { [Key(0)] public string X { get; set; } = ""; } + [MessagePackObject] public class HDup { [Key(0)] public List Items { get; set; } = new(); } + + [MessagePackObject] + [Union(0, typeof(NoIdSub))] + public abstract class NoIdBase { } + [MessagePackObject] public class NoIdSub : NoIdBase { [Key(0)] [CultElementId] public string Id { get; set; } = ""; } + [MessagePackObject] public class HAbstractNoId { [Key(0)] public List Items { get; set; } = new(); } + + [MessagePackObject] public class EDerived { [Key(0)] public int? Number { get; set; } [Key(1)] [CultElementId(nameof(Number))] public string Id { get; set; } = ""; } + [MessagePackObject] public class HDerived { [Key(0)] public List Items { get; set; } = new(); } + } +} diff --git a/tests/GameCult.Caching.Tests/ElementIdTests.cs b/tests/GameCult.Caching.Tests/ElementIdTests.cs new file mode 100644 index 00000000..aa1d7b57 --- /dev/null +++ b/tests/GameCult.Caching.Tests/ElementIdTests.cs @@ -0,0 +1,1425 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Reflection.Emit; +using System.Text.RegularExpressions; +using GameCult.Caching; +using GameCult.Caching.MessagePack; +using MessagePack; +using NUnit.Framework; + +namespace GameCult.Caching.Tests +{ + // C2a of docs/document-variants-cut.md: every object element of a list has an id, minted by the cache. + public class ElementIdTests + { + private static readonly CultDocumentRegistry Registry = CultDocumentRegistry.ForTypes(new[] { typeof(IdDeck) }); + + private static readonly ModuleBuilder Rejected = AssemblyBuilder + .DefineDynamicAssembly(new AssemblyName("RejectedElementShapes"), AssemblyBuilderAccess.Run) + .DefineDynamicModule("RejectedElementShapes"); + + private string _directory = string.Empty; + + [SetUp] + public void CreateDirectory() + { + _directory = Path.Combine(Path.GetTempPath(), $"cultlib-element-ids-{Guid.NewGuid():N}"); + Directory.CreateDirectory(_directory); + } + + [TearDown] + public void DeleteDirectory() + { + if (Directory.Exists(_directory)) + Directory.Delete(_directory, recursive: true); + } + + private string PathOf(string name) => Path.Combine(_directory, name); + + private static CultCache Open(string path) => + CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry }); + + private static IdDeck Deck(string name, int reels = 2) => new() + { + Name = name, + Reels = Enumerable.Range(0, reels).Select(_ => new IdReel { Label = "reel", Marks = { new IdMark { Text = "m" }, new IdMark { Text = "m" } } }).ToList() + }; + + private static string[] AllIds(IdDeck deck) => + deck.Reels.Select(reel => reel.Id) + .Concat(deck.Reels.SelectMany(reel => reel.Marks.Select(mark => mark.Id))) + .Concat(deck.Moves.Select(move => move.Id)) + .ToArray(); + + // A store written before ids existed: the current catalog and schema id, payloads without the id slots. + private static void WritePreIdStore(string path, string? keepKey, params (string Key, string Name, int Reels)[] records) + { + using (var seed = Open(path)) + { + seed.Commit(batch => batch.Upsert(typeof(IdDeck), Deck("seed"), new CultRecordKey("seed"))); + if (keepKey != null) + seed.Commit(batch => batch.Upsert(typeof(IdDeck), Deck("kept"), new CultRecordKey(keepKey))); + } + + var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + var schemaId = snapshot.Records.First().SchemaId; + var kept = snapshot.Records.Where(record => record.Key == keepKey); + snapshot.Records = kept.Concat(records.Select(record => new CultPersistedRecord + { + Key = record.Key, + SchemaId = schemaId, + StoredAt = DateTimeOffset.UtcNow.ToString("O"), + Payload = MessagePackSerializer.Serialize(new OldDeck + { + Name = record.Name, + Reels = Enumerable.Range(0, record.Reels).Select(_ => new OldReel { Label = "reel", Marks = { new OldMark { Text = "m" }, new OldMark { Text = "m" } } }).ToList() + }) + })).ToArray(); + // What a store from before ids declared. + snapshot.FormatVersion = CultPersistedStoreSnapshot.FormatV1; + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + } + + // An element id is an ordinary keyed string, so a record's bytes do not say whether it holds one: only the store header does. + // A cache whose type cannot see the ids (a later shape with no reels, declared compatible or resolved by name) that copies the + // record forward leaves the store marked, the record untouched, and its ids readable by the type that wrote them. + [Test] + public void ANarrowerTypeFlushingAnotherRecordKeepsAMarkedStoreMarked([Values] bool declared, [Values] bool directory) + { + var path = PathOf("narrow.cc"); + var deckSchemaId = Registry.GetRequired(typeof(IdDeck)).SchemaId; + var deck = Deck("r"); + using (var owner = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry, UseDirectoryStore = directory })) + owner.Commit(batch => batch.Upsert(typeof(IdDeck), deck, new CultRecordKey("r"))); + var marked = directory ? "cultcache.store.v5.directory-content-addressed-pages" : CultPersistedStoreSnapshot.FormatV3; + var before = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + Assert.That(before.FormatVersion, Is.EqualTo(marked)); + + var narrow = EmittedDocumentTypes.Emit( + "NarrowDeck" + (declared ? "Declared" : "ByName"), + "tests.element_id_deck", + declared ? "tests.element_id_deck.v9" : "tests.element_id_deck.v8", + new[] { new EmittedDocumentTypes.Field("Name", typeof(string), 0, IsName: true) }, + declared ? new[] { deckSchemaId } : null); + using (var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = CultDocumentRegistry.ForTypes(new[] { narrow }), UseDirectoryStore = directory })) + { + Assert.That(cache.Get(new CultRecordKey("r")), Is.Not.Null, "the narrower type reads the record it will copy"); + cache.UpsertAsync(narrow, EmittedDocumentTypes.New(narrow, ("Name", "y")), new CultRecordKey("y")).GetAwaiter().GetResult(); + cache.FlushAllBackingStores(); + } + + var after = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + Assert.That(after.FormatVersion, Is.EqualTo(marked)); + var kept = before.Records.Single(record => record.Key == "r"); + var copied = after.Records.Single(record => record.Key == "r"); + Assert.That($"{copied.SchemaId}|{copied.StoredAt}", Is.EqualTo($"{kept.SchemaId}|{kept.StoredAt}")); + using var reader = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry, UseDirectoryStore = directory }); + Assert.That(reader.Get(new CultRecordKey("r"))!.Reels.Select(reel => reel.Id), Is.EqualTo(deck.Reels.Select(reel => reel.Id))); + } + + private static CultPersistedRecord DiskRecord(string path, string key) => + CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)).Records.Single(record => record.Key == key); + + [Test] + public void ATypeWhoseListElementLacksAnIdIsRefusedByName() + { + var element = Rejected.DefineType("BareElement", TypeAttributes.Public); + element.SetCustomAttribute(new CustomAttributeBuilder(typeof(MessagePackObjectAttribute).GetConstructor(new[] { typeof(bool) })!, new object[] { false })); + element.DefineField("Label", typeof(string), FieldAttributes.Public) + .SetCustomAttribute(new CustomAttributeBuilder(typeof(KeyAttribute).GetConstructor(new[] { typeof(int) })!, new object[] { 0 })); + var elementType = element.CreateType()!; + + var document = Rejected.DefineType("BareDocument", TypeAttributes.Public); + document.SetCustomAttribute(new CustomAttributeBuilder(typeof(CultDocumentAttribute).GetConstructor(new[] { typeof(string), typeof(string) })!, new object[] { "shape.bare", "shape.bare.v1" })); + document.SetCustomAttribute(new CustomAttributeBuilder(typeof(MessagePackObjectAttribute).GetConstructor(new[] { typeof(bool) })!, new object[] { false })); + document.DefineField("Items", typeof(List<>).MakeGenericType(elementType), FieldAttributes.Public) + .SetCustomAttribute(new CustomAttributeBuilder(typeof(KeyAttribute).GetConstructor(new[] { typeof(int) })!, new object[] { 0 })); + + var refusal = Assert.Throws(() => CultDocumentRegistry.ForTypes(new[] { document.CreateType()! }))!; + Assert.That(refusal.Message, Does.Contain("BareDocument").And.Contain("BareElement").And.Contain("Items").And.Contain("[CultElementId]")); + } + + [Test] + public void AWriteMintsAnIdForEveryElementAndTheyRoundTrip() + { + var path = PathOf("deck.cc"); + var deck = Deck("d"); + deck.Moves.Add(new IdSlash { Reach = 1 }); + using (var writer = Open(path)) + writer.Commit(batch => batch.Upsert(typeof(IdDeck), deck, new CultRecordKey("d"))); + var written = AllIds(deck); + Assert.That(written, Has.Length.EqualTo(7)); + Assert.That(written, Has.All.Matches(id => Regex.IsMatch(id, "^[0-9a-f]{12}$"))); + + using var reader = Open(path); + Assert.That(AllIds(reader.Get(new CultRecordKey("d"))!), Is.EqualTo(written)); + } + + [Test] + public void TwoLookAlikeUnionElementsGetDistinctIds() + { + var deck = new IdDeck { Name = "u", Moves = { new IdSlash { Reach = 3 }, new IdSlash { Reach = 3 }, new IdStab { Depth = 3 } } }; + using var cache = Open(PathOf("u.cc")); + cache.Commit(batch => batch.Upsert(typeof(IdDeck), deck, new CultRecordKey("u"))); + Assert.That(deck.Moves.Select(move => move.Id).Distinct().Count(), Is.EqualTo(3)); + } + + [Test] + public void ADuplicateIdInOneListIsRefusedAndTheSameIdInTwoListsIsNot() + { + using var cache = Open(PathOf("dup.cc")); + var dup = Deck("dup"); + dup.Reels[0].Id = "aaaaaaaaaaaa"; + dup.Reels[1].Id = "aaaaaaaaaaaa"; + var refusal = Assert.Throws(() => cache.Commit(batch => batch.Upsert(typeof(IdDeck), dup, new CultRecordKey("dup"))))!; + Assert.That(refusal.Message, Does.Contain("aaaaaaaaaaaa")); + Assert.That(cache.Get(new CultRecordKey("dup")), Is.Null); + + var fine = Deck("fine"); + fine.Reels[0].Id = "aaaaaaaaaaaa"; + fine.Reels[0].Marks[0].Id = "aaaaaaaaaaaa"; + Assert.That(cache.Commit(batch => batch.Upsert(typeof(IdDeck), fine, new CultRecordKey("fine"))), Is.True); + } + + [Test] + public void APreIdStoreLoadsAndItsIdsAreMintedExactlyOnce() + { + var path = PathOf("old.cc"); + WritePreIdStore(path, null, ("a", "a", 2), ("b", "b", 1)); + var before = DiskRecord(path, "a").Payload; + Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v1"), "the fixture is a store from before ids"); + + string[] first; + using (var cache = Open(path)) + { + first = AllIds(cache.Get(new CultRecordKey("a"))!); + Assert.That(first, Has.Length.EqualTo(6)); + Assert.That(first, Has.All.Not.Empty); + Assert.That(first.Distinct().Count(), Is.EqualTo(first.Length)); + Assert.That(DiskRecord(path, "a").Payload, Is.EqualTo(before), "loading mints in memory and writes nothing"); + cache.Commit(batch => batch.Upsert(typeof(IdDeck), Deck("other"), new CultRecordKey("other"))); + Assert.That(DiskRecord(path, "a").Payload, Is.EqualTo(before), "a write of another record leaves a's bytes as they are: its minted ids stay in memory"); + Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v3"), "and the write declares that the store holds ids: the record it wrote holds some"); + } + + using (var again = Open(path)) + Assert.That(AllIds(again.Get(new CultRecordKey("a"))!), Is.EqualTo(first), "a reload is identical to what was read"); + + using (var writer = Open(path)) + { + var deck = writer.Get(new CultRecordKey("a"))!; + deck.Name = "a2"; + writer.Commit(batch => batch.Upsert(typeof(IdDeck), deck, new CultRecordKey("a"))); + writer.Commit(batch => batch.Upsert(typeof(IdDeck), deck, new CultRecordKey("a"))); + } + + using var last = Open(path); + var reread = last.Get(new CultRecordKey("a"))!; + Assert.That(reread.Name, Is.EqualTo("a2")); + Assert.That(AllIds(reread), Is.EqualTo(first), "the ids minted at load are the ids written, and stay through repeated writes"); + } + + [Test] + public void MintElementIdsRewritesEveryPreIdRecordAndNothingElse() + { + var path = PathOf("mint.cc"); + WritePreIdStore(path, "new", ("old1", "o1", 1), ("old2", "o2", 3)); + string[] ids1, ids2; + var newRecord = DiskRecord(path, "new"); + + using (var cache = Open(path)) + { + ids1 = AllIds(cache.Get(new CultRecordKey("old1"))!); + ids2 = AllIds(cache.Get(new CultRecordKey("old2"))!); + Assert.That(cache.MintElementIds(), Is.EqualTo(2)); + Assert.That(cache.MintElementIds(), Is.EqualTo(0), "the second run finds nothing to do"); + } + + Assert.That(DiskRecord(path, "new").Payload, Is.EqualTo(newRecord.Payload)); + Assert.That(DiskRecord(path, "new").StoredAt, Is.EqualTo(newRecord.StoredAt), "a record that already had ids is not touched"); + Assert.That(AllIds(MessagePackSerializer.Deserialize(DiskRecord(path, "old1").Payload)), Is.EqualTo(ids1), "the ids are on disk now"); + using var reader = Open(path); + Assert.That(AllIds(reader.Get(new CultRecordKey("old1"))!), Is.EqualTo(ids1)); + Assert.That(AllIds(reader.Get(new CultRecordKey("old2"))!), Is.EqualTo(ids2)); + Assert.That(reader.MintElementIds(), Is.EqualTo(0)); + } + + // A pre-id record this cache reads whole through the catalog's schema name is owned: the rewrite that persists its minted + // ids is accepted and stores it under this type's id. + [Test] + public void MintElementIdsRewritesAPreIdRecordUnderAnotherRuntimesIdThatItReadsWhole() + { + const string otherRuntimeId = "other.runtime.id_deck"; + var path = PathOf("mint-drift.cc"); + WritePreIdStore(path, null, ("a", "a", 1)); + var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + var entry = snapshot.SchemaCatalog.Single(); + entry.SchemaId = otherRuntimeId; + entry.ContentHash = otherRuntimeId; + entry.CompatibleSchemaIds = new[] { otherRuntimeId }; + snapshot.Records.Single().SchemaId = otherRuntimeId; + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + + using (var cache = Open(path)) + { + Assert.That(cache.MintElementIds(), Is.EqualTo(1)); + Assert.That(cache.GetStored(new CultRecordKey("a"))!.StoredSchemaId, Is.Not.EqualTo(otherRuntimeId), "the rewrite restamped it"); + } + + Assert.That(DiskRecord(path, "a").SchemaId, Is.Not.EqualTo(otherRuntimeId)); + } + + // A pre-id record this cache reads only by dropping a persisted slot is read-only, so the rewrite that would persist its + // minted ids is refused typed, naming it, and the store is left byte for byte. + [Test] + public void MintElementIdsIsRefusedByAReadOnlyRecordWithMintedIds() + { + const string otherRuntimeId = "other.runtime.id_deck"; + var path = PathOf("mint-read-only.cc"); + WritePreIdStore(path, null, ("a", "a", 1)); + var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + var entry = snapshot.SchemaCatalog.Single(); + entry.SchemaId = otherRuntimeId; + entry.ContentHash = otherRuntimeId; + entry.CompatibleSchemaIds = new[] { otherRuntimeId }; + entry.Members = entry.Members.Append(new CultSchemaMemberCatalogEntry { Slot = 99, MemberName = "Dropped", TypeName = "System.String" }).ToArray(); + snapshot.Records.Single().SchemaId = otherRuntimeId; + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + var before = File.ReadAllBytes(path); + + using (var cache = Open(path)) + { + Assert.That(AllIds(cache.Get(new CultRecordKey("a"))!), Has.All.Not.Empty, "the load minted ids in memory"); + var refusal = Assert.Throws(() => cache.MintElementIds())!; + Assert.That((refusal.RecordKey, refusal.SchemaId), Is.EqualTo(("a", otherRuntimeId))); + cache.FlushAsync().GetAwaiter().GetResult(); + } + + Assert.That(File.ReadAllBytes(path), Is.EqualTo(before)); + } + + [Test] + public void TheByteCostOfAnIdIsMeasured() + { + var ids = new IdDeck { Name = "m" }; + var old = new OldDeck { Name = "m" }; + for (var index = 0; index < 100; index++) + { + ids.Reels.Add(new IdReel { Label = "reel", Id = "0123456789ab" }); + old.Reels.Add(new OldReel { Label = "reel" }); + } + + // IdDeck also carries the empty Moves list at slot 2: one byte, once. + var perElement = (MessagePackSerializer.Serialize(ids).Length - MessagePackSerializer.Serialize(old).Length - 1) / 100.0; + TestContext.Out.WriteLine($"element id cost: {perElement} bytes per element (12-char id in the next free slot)"); + Assert.That(perElement, Is.EqualTo(13.0), "1 byte of string header and 12 of id; the element's array header does not grow"); + } + + [Test] + public void ARemovedPreIdRecordIsNotRewrittenByMintElementIds() + { + var path = PathOf("removed.cc"); + WritePreIdStore(path, null, ("a", "a", 1), ("b", "b", 1)); + using var cache = Open(path); + // Conditional, so the file is merged onto and b's ids stay in memory only. + cache.Commit(batch => + { + batch.Expect(new CultRecordKey("elsewhere"), null); + batch.Remove(new CultRecordKey("a")); + }); + Assert.That(cache.MintElementIds(), Is.EqualTo(1), "only the survivor is listed; the removed record is not resurrected"); + Assert.That(cache.Get(new CultRecordKey("a")), Is.Null); + } + + [Test] + public void ADerivedIdIsTheSourceTextNeverRandomAndItsDuplicatesAreRefused() + { + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(SpanDoc) }); + SpanDoc Make() => new() { Name = "s", Spans = { new IdSpan { Offset = 0, Size = 4 }, new IdSpan { Offset = 262144, Size = 4 } } }; + var first = Make(); + var second = Make(); + using (var cache = CultCacheMessagePack.Create(PathOf("span1.cc"), new CultCacheOpenOptions { Registry = registry })) + cache.Commit(batch => batch.Upsert(typeof(SpanDoc), first, new CultRecordKey("s"))); + using (var cache = CultCacheMessagePack.Create(PathOf("span2.cc"), new CultCacheOpenOptions { Registry = registry })) + cache.Commit(batch => batch.Upsert(typeof(SpanDoc), second, new CultRecordKey("s"))); + Assert.That(first.Spans.Select(span => span.Id), Is.EqualTo(new[] { "0", "262144" })); + Assert.That(MessagePackSerializer.Serialize(second), Is.EqualTo(MessagePackSerializer.Serialize(first)), "the same content encodes to the same bytes"); + + var twice = new SpanDoc { Name = "t", Spans = { new IdSpan { Offset = 7 }, new IdSpan { Offset = 7 } } }; + using var refusing = CultCacheMessagePack.Create(PathOf("span3.cc"), new CultCacheOpenOptions { Registry = registry }); + var refusal = Assert.Throws(() => refusing.Commit(batch => batch.Upsert(typeof(SpanDoc), twice, new CultRecordKey("t"))))!; + Assert.That(refusal.Message, Does.Contain("'7'")); + } + + // ---- C2a fix batch: the marker, refusals at load, override values, atomic minting ---- + + private const string HexA = "aaaaaaaaaaaa"; + + private static CultCache OpenWith(string path, bool directory, params Type[] types) => + CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = CultDocumentRegistry.ForTypes(types), UseDirectoryStore = directory }); + + private static string HeaderOf(string path) => + CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)).FormatVersion; + + private static void UpsertPlain(CultCache cache, string key) => + cache.Commit(batch => batch.Upsert(typeof(PreCut2FixtureItem), new PreCut2FixtureItem { Name = key }, new CultRecordKey(key))); + + private static CultElementIdException Refusal(Exception error) + { + for (var inner = error; inner != null; inner = inner.InnerException!) + { + if (inner is CultElementIdException typed) + return typed; + } + + throw new AssertionException("no CultElementIdException in the chain: " + error); + } + + private static byte[] OldShapedReels(string label) => + MessagePackSerializer.Serialize(new List { new() { Label = label, Marks = { new OldMark { Text = "x" }, new OldMark { Text = "y" } } } }); + + // A holder the walk does not own (ImmutableArray here) is one whole value: its elements carry no element ids, so it never + // marks the store, and a variant replaces it whole (q3 amended 2026-10-16). + [Test] + public void AWholeListHolderNeverMarksTheStoreAndAVariantReplacesItWhole() + { + Assert.That(CultElementIds.CanHoldIds(typeof(IdWholeHolder)), Is.False); + var path = PathOf("whole-holder.cc"); + using (var cache = OpenWith(path, false, typeof(IdWholeHolder))) + { + cache.Commit(batch => batch.Upsert(typeof(IdWholeHolder), + new IdWholeHolder { Name = "base", Reels = ImmutableArray.Create(new IdReel { Label = "set", Id = HexA }, new IdReel { Label = "unset" }) }, + new CultRecordKey("base"))); + } + + Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v1"), "ids inside a whole value mark nothing"); + + using (var fresh = OpenWith(path, false, typeof(IdWholeHolder))) + { + fresh.Commit(batch => batch.Upsert(typeof(IdWholeHolder), new IdWholeHolder { Name = "other" }, new CultRecordKey("other"))); + var back = fresh.Get(new CultRecordKey("base"))!; + Assert.That(back.Reels.Select(reel => reel.Id), Is.EqualTo(new[] { HexA, "" }), "set ids are plain values, unset ids stay unset"); + } + + Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v1")); + using (var cache = OpenWith(path, false, typeof(IdWholeHolder))) + { + var replacement = ImmutableArray.Create(new IdReel { Label = "v", Id = "eeeeeeeeeeee" }); + cache.Commit(batch => batch.UpsertVariant(new CultRecordKey("v"), new CultRecordKey("base"), + new[] { cache.Override(nameof(IdWholeHolder.Name), "v"), cache.Override(nameof(IdWholeHolder.Reels), replacement) })); + Assert.That(cache.Get(new CultRecordKey("v"))!.Reels.Select(reel => reel.Id), Is.EqualTo(new[] { "eeeeeeeeeeee" }), "replaced whole"); + Assert.That(cache.Get(new CultRecordKey("base"))!.Reels, Has.Length.EqualTo(2), "the base is untouched"); + } + + Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v2"), "a variant makes v2, and nothing in it holds element ids"); + using var again = OpenWith(path, false, typeof(IdWholeHolder)); + Assert.That(again.Get(new CultRecordKey("v"))!.Reels.Single().Id, Is.EqualTo("eeeeeeeeeeee")); + + var step = new CultVariantOverride(CultOverrideOp.Set, new[] { new CultPathStep(1, HexA) }, string.Empty, MessagePackSerializer.Serialize("x")); + var refused = Assert.Catch(() => again.Commit(batch => batch.UpsertVariant(new CultRecordKey("p"), new CultRecordKey("base"), new[] { again.Override(nameof(IdWholeHolder.Name), "p"), step })))!; + Assert.That(refused.ToString(), Does.Contain("top-level Set overrides only")); + } + + [Test] + public void AStoreThatCanHoldElementIdsCarriesTheV3MarkerAndOneThatCannotStaysV1() + { + var ids = PathOf("ids.cc"); + using (var cache = OpenWith(ids, false, typeof(IdDeck))) + cache.Commit(batch => batch.Upsert(typeof(IdDeck), Deck("d"), new CultRecordKey("d"))); + Assert.That(HeaderOf(ids), Is.EqualTo("cultcache.store.v3")); + + var plain = PathOf("plain.cc"); + using (var cache = OpenWith(plain, false, typeof(PreCut2FixtureItem))) + UpsertPlain(cache, "a"); + Assert.That(HeaderOf(plain), Is.EqualTo("cultcache.store.v1")); + + using var reread = OpenWith(ids, false, typeof(IdDeck)); + Assert.That(reread.Get(new CultRecordKey("d")), Is.Not.Null, "a v3 store reads"); + } + + [Test] + public void AStagedWriteThatIsFlushedCarriesTheMarkerAndLeavesLoadMintedIdsToMintElementIds() + { + var path = PathOf("flush.cc"); + WritePreIdStore(path, null, ("a", "a", 1)); + using (var cache = Open(path)) + { + cache.UpsertAsync(typeof(IdDeck), Deck("staged"), new CultRecordKey("staged")).GetAwaiter().GetResult(); + cache.FlushAllBackingStores(); + Assert.That(cache.MintElementIds(), Is.EqualTo(1), "the flush wrote only the staged record: a's minted ids are still in memory"); + } + + Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v3")); + Assert.That(AllIds(MessagePackSerializer.Deserialize(DiskRecord(path, "a").Payload)), Has.All.Not.Empty); + } + + [Test] + public void AnUnconditionalWriteOfAPlainRecordKeepsTheMarkerWhileTheStoreHoldsIds() + { + var path = PathOf("wholeview.cc"); + using var cache = OpenWith(path, false, typeof(IdDeck), typeof(PreCut2FixtureItem)); + cache.Commit(batch => batch.Upsert(typeof(IdDeck), Deck("d"), new CultRecordKey("d"))); + UpsertPlain(cache, "plain"); + Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v3"), "the store still holds a deck, whatever this commit wrote"); + } + + [Test] + public void AMergeOntoAFileAnotherWriterMarkedKeepsTheMarkerEvenWhenThisWriterHoldsNoIds() + { + var path = PathOf("merge.cc"); + Type[] types = { typeof(IdDeck), typeof(PreCut2FixtureItem) }; + using var first = OpenWith(path, false, types); + UpsertPlain(first, "first"); + using var second = OpenWith(path, false, types); + Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v1")); + first.Commit(batch => batch.Upsert(typeof(IdDeck), Deck("d"), new CultRecordKey("d"))); + Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v3")); + + Assert.That(second.Commit(batch => + { + batch.Expect(new CultRecordKey("second"), null); + batch.Upsert(typeof(PreCut2FixtureItem), new PreCut2FixtureItem { Name = "second" }, new CultRecordKey("second")); + }), Is.True); + Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v3"), "the file holds a deck another writer put there; second holds none"); + } + + [Test] + public void AnUnconditionalWriteOfAPlainRecordDoesNotPersistOrMarkTheIdsALoadedRecordMinted() + { + var path = PathOf("loaded-minted.cc"); + WritePreIdStore(path, null, ("a", "a", 1)); + var before = DiskRecord(path, "a"); + using (var cache = OpenWith(path, false, typeof(IdDeck), typeof(PreCut2FixtureItem))) + { + Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v1")); + UpsertPlain(cache, "p"); + Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v1"), "nothing the write stored holds an id"); + Assert.That(DiskRecord(path, "a").Payload, Is.EqualTo(before.Payload), "a is copied as it was stored"); + cache.MintElementIds(); + } + + Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v3"), "MintElementIds persisted a's minted ids"); + Assert.That(AllIds(MessagePackSerializer.Deserialize(DiskRecord(path, "a").Payload)), Has.All.Not.Empty); + } + + [Test] + public void AMarkedFileStaysMarkedWhenACommitOnlyTouchesOtherRecords() + { + var path = PathOf("sticky.cc"); + using (var cache = OpenWith(path, false, typeof(IdDeck), typeof(PreCut2FixtureItem))) + cache.Commit(batch => batch.Upsert(typeof(IdDeck), Deck("d"), new CultRecordKey("d"))); + using (var cache = OpenWith(path, false, typeof(IdDeck), typeof(PreCut2FixtureItem))) + { + Assert.That(cache.Commit(batch => + { + batch.Expect(new CultRecordKey("plain"), null); + batch.Upsert(typeof(PreCut2FixtureItem), new PreCut2FixtureItem { Name = "plain" }, new CultRecordKey("plain")); + }), Is.True); + } + + Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v3"), "a merge onto a marked file does not shed the marker"); + } + + [Test] + public void ADirectoryStoreThatCanHoldElementIdsCarriesAV5ManifestAndStaysMarked() + { + var ids = PathOf("ids-dir.cc"); + using (var cache = OpenWith(ids, true, typeof(IdDeck), typeof(PreCut2FixtureItem))) + cache.Commit(batch => batch.Upsert(typeof(IdDeck), Deck("d"), new CultRecordKey("d"))); + Assert.That(HeaderOf(ids), Is.EqualTo("cultcache.store.v5.directory-content-addressed-pages")); + using (var cache = OpenWith(ids, true, typeof(IdDeck), typeof(PreCut2FixtureItem))) + { + Assert.That(cache.Get(new CultRecordKey("d")), Is.Not.Null, "a v5 manifest reads"); + UpsertPlain(cache, "plain"); + } + + Assert.That(HeaderOf(ids), Is.EqualTo("cultcache.store.v5.directory-content-addressed-pages"), "a rewrite that touches a plain record keeps the marker"); + + var mixed = PathOf("mixed-dir.cc"); + using (var cache = OpenWith(mixed, true, typeof(IdDeck), typeof(PreCut2FixtureItem))) + { + cache.Commit(batch => + { + batch.Upsert(typeof(PreCut2FixtureItem), new PreCut2FixtureItem { Name = "p" }, new CultRecordKey("p")); + batch.Upsert(typeof(IdDeck), Deck("d"), new CultRecordKey("d")); + }); + } + + Assert.That(HeaderOf(mixed), Is.EqualTo("cultcache.store.v5.directory-content-addressed-pages"), "one written record that can hold ids is enough"); + + var plain = PathOf("plain-dir.cc"); + using (var cache = OpenWith(plain, true, typeof(PreCut2FixtureItem))) + UpsertPlain(cache, "a"); + Assert.That(HeaderOf(plain), Is.EqualTo("cultcache.store.v4.directory-content-addressed-pages")); + } + + [Test] + public void AVariantStoreThatCanHoldElementIdsIsV3AndItsVariantsRead() + { + var path = PathOf("variant-v3.cc"); + using (var cache = OpenWith(path, false, typeof(IdDeck))) + { + cache.Commit(batch => + { + batch.Upsert(typeof(IdDeck), Deck("base"), new CultRecordKey("base")); + batch.UpsertVariant(new CultRecordKey("v"), new CultRecordKey("base"), new[] { cache.Override(nameof(IdDeck.Name), "v") }); + }); + } + + Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v3")); + using var again = OpenWith(path, false, typeof(IdDeck)); + Assert.That(again.Get(new CultRecordKey("v"))!.Name, Is.EqualTo("v")); + } + + [Test] + public void ADuplicateIdOnDiskRefusesTheLoadNamingTheRecordTheListAndTheId() + { + var path = PathOf("dup-disk.cc"); + WritePreIdStore(path, null, ("a", "a", 1)); + var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + var deck = Deck("dup"); + deck.Reels[0].Id = HexA; + deck.Reels[1].Id = HexA; + deck.Reels[0].Marks[0].Id = HexA; + snapshot.Records[0].Payload = MessagePackSerializer.Serialize(deck); + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + + var refusal = Refusal(Assert.Catch(() => Open(path).Dispose())!); + Assert.That(refusal.RecordKey, Is.EqualTo("a")); + Assert.That(refusal.ElementId, Is.EqualTo(HexA)); + Assert.That(refusal.ListPath, Is.EqualTo("a.1")); + } + + [Test] + public void AnIdOfTheWrongFormatOnDiskRefusesTheLoadAndAWriteOfItIsRefused() + { + var path = PathOf("bad-format.cc"); + WritePreIdStore(path, null, ("a", "a", 1)); + var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + var deck = Deck("bad"); + deck.Reels[0].Id = "t00000000001"; + snapshot.Records[0].Payload = MessagePackSerializer.Serialize(deck); + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + var refusal = Refusal(Assert.Catch(() => Open(path).Dispose())!); + Assert.That(refusal.RecordKey, Is.EqualTo("a")); + Assert.That(refusal.ElementId, Is.EqualTo("t00000000001")); + + using var cache = Open(PathOf("bad-write.cc")); + var written = Deck("w"); + written.Reels[0].Marks[1].Id = "NOTHEX"; + Assert.That(Assert.Throws(() => cache.Commit(batch => batch.Upsert(typeof(IdDeck), written, new CultRecordKey("w"))))!.ElementId, Is.EqualTo("NOTHEX")); + } + + [Test] + public void ABatchRefusedForADuplicateLeavesEveryDocumentItHeldUnminted() + { + using var cache = Open(PathOf("batch-refused.cc")); + var fine = Deck("fine"); + var dup = Deck("dup"); + dup.Reels[0].Id = HexA; + dup.Reels[1].Id = HexA; + Assert.Throws(() => cache.Commit(batch => + { + batch.Upsert(typeof(IdDeck), fine, new CultRecordKey("fine")); + batch.Upsert(typeof(IdDeck), dup, new CultRecordKey("dup")); + })); + Assert.That(AllIds(fine), Has.All.Empty, "the refused batch minted nothing into the other document"); + Assert.That(dup.Reels[0].Marks.Select(mark => mark.Id), Has.All.Empty); + Assert.That(cache.Get(new CultRecordKey("fine")), Is.Null); + } + + [Test] + public void AWriteThatDoesNotCommitGivesItsIdsBack() + { + using var cache = Open(PathOf("mismatch.cc")); + cache.Commit(batch => batch.Upsert(typeof(IdDeck), Deck("existing"), new CultRecordKey("existing"))); + var fresh = Deck("fresh"); + var outcome = cache.TryCommit(batch => + { + batch.Expect(new CultRecordKey("existing"), null); + batch.Upsert(typeof(IdDeck), fresh, new CultRecordKey("fresh")); + }); + Assert.That(outcome, Is.EqualTo(CultCommitOutcome.Mismatch)); + Assert.That(AllIds(fresh), Has.All.Empty); + } + + [Test] + public void AnUnconditionalWriteLeavesTheLoadMintedIdsOfTheRecordsItDidNotStageOnTheList() + { + var path = PathOf("unconditional.cc"); + WritePreIdStore(path, null, ("a", "a", 1), ("b", "b", 1)); + using var cache = Open(path); + cache.Commit(batch => batch.Upsert(typeof(IdDeck), Deck("other"), new CultRecordKey("other"))); + Assert.That(AllIds(MessagePackSerializer.Deserialize(DiskRecord(path, "b").Payload)), Has.All.Empty, "the write did not touch b"); + Assert.That(cache.MintElementIds(), Is.EqualTo(2), "a and b exist with ids only in memory until MintElementIds writes them"); + Assert.That(AllIds(MessagePackSerializer.Deserialize(DiskRecord(path, "b").Payload)), Has.All.Not.Empty); + } + + [Test] + public void AConditionalWriteThatLeavesOtherRecordsAsTheyWereLeavesThemOnTheList() + { + var path = PathOf("conditional.cc"); + WritePreIdStore(path, null, ("a", "a", 1), ("b", "b", 1)); + using var cache = Open(path); + Assert.That(cache.Commit(batch => + { + batch.Expect(new CultRecordKey("fresh"), null); + batch.Upsert(typeof(IdDeck), Deck("fresh"), new CultRecordKey("fresh")); + }), Is.True); + Assert.That(cache.MintElementIds(), Is.EqualTo(2), "a and b were not written; their ids are still only in memory"); + } + + [Test] + public void AnOverrideValueGetsItsIdsMintedOnWriteWhoeverBuiltIt() + { + var path = PathOf("override-write.cc"); + using (var cache = Open(path)) + { + var reels = new List { new() { Label = "v", Marks = { new IdMark { Text = "x" } } }, new() { Label = "v" } }; + cache.Commit(batch => + { + batch.Upsert(typeof(IdDeck), Deck("base"), new CultRecordKey("base")); + // One built by the cache's own helper, one from bytes the id rule never saw. + batch.UpsertVariant(new CultRecordKey("helper"), new CultRecordKey("base"), + new[] { cache.Override(nameof(IdDeck.Name), "helper"), cache.Override(nameof(IdDeck.Reels), reels) }); + batch.UpsertVariant(new CultRecordKey("raw"), new CultRecordKey("base"), + new[] { cache.Override(nameof(IdDeck.Name), "raw"), CultVariantOverride.Set(1, OldShapedReels("r")) }); + }); + Assert.That(reels.Select(reel => reel.Id), Has.All.Empty, "the caller's list is not written to; ids are minted in the admitted copy"); + foreach (var key in new[] { "helper", "raw" }) + { + var resolved = cache.Get(new CultRecordKey(key))!; + var ids = resolved.Reels.Select(reel => reel.Id).Concat(resolved.Reels.SelectMany(reel => reel.Marks.Select(mark => mark.Id))).ToArray(); + Assert.That(ids, Has.All.Matches(id => Regex.IsMatch(id, "^[0-9a-f]{12}$")), key); + } + } + + foreach (var key in new[] { "helper", "raw" }) + { + var stored = MessagePackSerializer.Deserialize>(DiskRecord(path, key).Variant!.Overrides.Single(entry => entry.Path[0].Slot == 1).Value); + Assert.That(stored.Select(reel => reel.Id), Has.All.Matches(id => id.Length == 12), $"{key}: the persisted delta carries the ids"); + using var reader = Open(path); + Assert.That(reader.Get(new CultRecordKey(key))!.Reels.Select(reel => reel.Id), Is.EqualTo(stored.Select(reel => reel.Id))); + } + } + + [Test] + public void AnOverrideValueWithADuplicateIdIsRefusedNamingTheVariant() + { + using var cache = Open(PathOf("override-dup.cc")); + cache.Commit(batch => batch.Upsert(typeof(IdDeck), Deck("base"), new CultRecordKey("base"))); + var dup = new List { new() { Id = HexA }, new() { Id = HexA } }; + var refusal = Assert.Throws(() => cache.Commit(batch => + batch.UpsertVariant(new CultRecordKey("v"), new CultRecordKey("base"), + new[] { cache.Override(nameof(IdDeck.Name), "v"), cache.Override(nameof(IdDeck.Reels), dup) })))!; + Assert.That(refusal.RecordKey, Does.StartWith("v")); + Assert.That(refusal.ElementId, Is.EqualTo(HexA)); + Assert.That(cache.Get(new CultRecordKey("v")), Is.Null); + } + + [Test] + public void AVariantWrittenBeforeIdsMintsItsOverrideIdsOnLoadAndMintElementIdsRewritesIt() + { + var path = PathOf("override-load.cc"); + using (var seed = Open(path)) + { + seed.Commit(batch => + { + batch.Upsert(typeof(IdDeck), Deck("base"), new CultRecordKey("base")); + batch.UpsertVariant(new CultRecordKey("v"), new CultRecordKey("base"), + new[] { seed.Override(nameof(IdDeck.Name), "v"), seed.Override(nameof(IdDeck.Reels), new List { new() }) }); + }); + } + + var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + var record = snapshot.Records.Single(entry => entry.Key == "v"); + record.Variant = new CultVariantDelta(record.Variant!.BaseKey, record.Variant.Overrides + .Select(entry => entry.Path[0].Slot == 1 ? CultVariantOverride.Set(1, OldShapedReels("old")) : entry).ToArray()); + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + var untouched = File.ReadAllBytes(path); + + string[] first; + using (var peek = Open(path)) + first = AllIds(peek.Get(new CultRecordKey("v"))!); + using (var cache = Open(path)) + { + Assert.That(AllIds(cache.Get(new CultRecordKey("v"))!), Is.EqualTo(first), "the ids minted at load are a function of the record, so a second load mints the same"); + Assert.That(first, Has.Length.EqualTo(3)); + Assert.That(first, Has.All.Not.Empty); + Assert.That(File.ReadAllBytes(path), Is.EqualTo(untouched), "loading mints in memory and writes nothing"); + cache.Commit(batch => batch.Upsert(typeof(IdDeck), Deck("other"), new CultRecordKey("other"))); + // A base edit re-resolves the variant as a dependent; it is still only in memory. + var baseDeck = cache.Get(new CultRecordKey("base"))!; + baseDeck.Reels[0].Label = "edited"; + cache.Commit(batch => batch.Upsert(typeof(IdDeck), baseDeck, new CultRecordKey("base"))); + Assert.That(AllIds(cache.Get(new CultRecordKey("v"))!), Is.EqualTo(first), "the dependent re-resolves to the same minted ids"); + Assert.That(cache.MintElementIds(), Is.EqualTo(1), "a flush of the store persisted the delta as it was handed over, so the variant is still in memory only"); + Assert.That(cache.MintElementIds(), Is.EqualTo(0)); + } + + var persisted = MessagePackSerializer.Deserialize>(DiskRecord(path, "v").Variant!.Overrides.Single(entry => entry.Path[0].Slot == 1).Value); + Assert.That(persisted.Select(reel => reel.Id).Concat(persisted.SelectMany(reel => reel.Marks.Select(mark => mark.Id))), Is.EqualTo(first)); + using var again = Open(path); + Assert.That(AllIds(again.Get(new CultRecordKey("v"))!), Is.EqualTo(first)); + } + + [Test] + public void ARefusalAfterMintingLeavesEveryPlainDocumentInTheBatchUnminted() + { + using var cache = Open(PathOf("late-refusal.cc")); + cache.Commit(batch => batch.Upsert(typeof(IdDeck), Deck("base"), new CultRecordKey("base"))); + var plain = Deck("plain"); + var dup = new List { new() { Id = HexA }, new() { Id = HexA } }; + Assert.Throws(() => cache.Commit(batch => + { + batch.Upsert(typeof(IdDeck), plain, new CultRecordKey("plain")); + batch.UpsertVariant(new CultRecordKey("v"), new CultRecordKey("base"), + new[] { cache.Override(nameof(IdDeck.Name), "v"), cache.Override(nameof(IdDeck.Reels), dup) }); + })); + Assert.That(AllIds(plain), Has.All.Empty, "the variant's refusal came after the plain record was minted; the mint was given back"); + Assert.That(cache.Get(new CultRecordKey("plain")), Is.Null); + } + + // ---- C2a fix batch 3: mark by content, one object twice ---- + + private static IdDeck EmptyDeck(string name) => new() { Name = name }; + + [Test] + public void ASingleFileStoreIsMarkedByTheIdsItHoldsNotByTheTypesItCouldHold() + { + var empty = PathOf("empty-deck.cc"); + using (var cache = Open(empty)) + cache.Commit(batch => batch.Upsert(typeof(IdDeck), EmptyDeck("e"), new CultRecordKey("e"))); + Assert.That(HeaderOf(empty), Is.EqualTo("cultcache.store.v1"), "a deck with no elements holds no id"); + + var flushed = PathOf("flushed.cc"); + using (var cache = Open(flushed)) + { + cache.UpsertAsync(typeof(IdDeck), EmptyDeck("e"), new CultRecordKey("e")).GetAwaiter().GetResult(); + cache.FlushAllBackingStores(); + } + + Assert.That(HeaderOf(flushed), Is.EqualTo("cultcache.store.v1"), "the whole-store flush decides by content too"); + + using (var cache = Open(flushed)) + { + cache.UpsertAsync(typeof(IdDeck), Deck("d"), new CultRecordKey("d")).GetAwaiter().GetResult(); + cache.FlushAllBackingStores(); + } + + Assert.That(HeaderOf(flushed), Is.EqualTo("cultcache.store.v3"), "one element with an id marks the flush"); + + var marked = PathOf("marked.cc"); + using (var cache = Open(marked)) + cache.Commit(batch => batch.Upsert(typeof(IdDeck), Deck("d"), new CultRecordKey("d"))); + using (var cache = Open(marked)) + { + Assert.That(cache.Commit(batch => + { + batch.Expect(new CultRecordKey("e"), null); + batch.Upsert(typeof(IdDeck), EmptyDeck("e"), new CultRecordKey("e")); + }), Is.True); + } + + Assert.That(HeaderOf(marked), Is.EqualTo("cultcache.store.v3"), "a file already marked on disk stays marked"); + } + + [Test] + public void AnIdAlreadyThereMarksTheStoreAndSoDoesARecordLoadedHoldingOne() + { + var preset = PathOf("preset.cc"); + using (var cache = Open(preset)) + cache.Commit(batch => batch.Upsert(typeof(IdDeck), new IdDeck { Name = "p", Reels = { new IdReel { Id = HexA } } }, new CultRecordKey("p"))); + Assert.That(HeaderOf(preset), Is.EqualTo("cultcache.store.v3"), "the deck minted nothing, and still holds an id"); + + // The deck reloads holding ids it did not mint; a whole-store flush of a store now holding only that deck and an empty one. + using (var cache = Open(preset)) + { + cache.UpsertAsync(typeof(IdDeck), EmptyDeck("e"), new CultRecordKey("e")).GetAwaiter().GetResult(); + cache.FlushAllBackingStores(); + } + + Assert.That(HeaderOf(preset), Is.EqualTo("cultcache.store.v3"), "a loaded record's ids count"); + } + + // A record a caller builds and hands straight to a store never passes an admission; the flag still means what it persists. + [TestCase(false, "cultcache.store.v3")] + [TestCase(true, "cultcache.store.v5.directory-content-addressed-pages")] + public void ARecordPushedStraightToAStoreIsMarkedByTheIdsItHolds(bool directory, string marked) + { + var path = PathOf(directory ? "direct-dir.cc" : "direct.cc"); + using var cache = OpenWith(path, directory, typeof(IdDeck)); + var store = cache.BackingStores[0]; + var descriptor = Registry.GetRequired(); + var withId = new IdDeck { Name = "d", Reels = { new IdReel { Id = HexA } } }; + + store.Push(new CultStoredDocument(new CultRecordKey("d"), "2026-09-29T00:00:00.0000000+00:00", descriptor, withId)); + store.PushAll(); + Assert.That(HeaderOf(path), Is.EqualTo(marked)); + + // The directory store writes only its dirty pages and keeps a marked manifest; the single file is whole-store. + if (directory) return; + store.Push(new CultStoredDocument(new CultRecordKey("d"), "2026-09-29T00:00:00.0000000+00:00", descriptor, EmptyDeck("d"))); + store.PushAll(); + Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v1"), "a whole-store flush decides by what the store now holds"); + } + + // Soul's P7 and P10: a store reads the header off what it writes, when it writes it. A staged document changed after it was + // staged, loaded or not, is marked by what it holds at the flush, in both directions. A document changed in place and not + // staged is not written, so it marks nothing. + [TestCase(false)] + [TestCase(true)] + public void AFlushIsMarkedByWhatAStagedDocumentHoldsNowNotWhatItHeldWhenStaged(bool loaded) + { + var path = PathOf(loaded ? "mutated-loaded.cc" : "mutated-staged.cc"); + var key = new CultRecordKey("d"); + using (var seed = Open(path)) + seed.Commit(batch => batch.Upsert(typeof(IdDeck), EmptyDeck("d"), key)); + Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v1")); + var seeded = DiskRecord(path, "d"); + + using var cache = Open(path); + var deck = loaded ? cache.Get(key)! : EmptyDeck("d"); + void Stage() => cache.UpsertAsync(typeof(IdDeck), deck, key).GetAwaiter().GetResult(); + if (!loaded) + Stage(); + + deck.Reels.Add(new IdReel { Id = HexA }); + if (loaded) + { + cache.BackingStores[0].PushAll(); + Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v1"), "a change in place that was never staged is not written"); + Assert.That(DiskRecord(path, "d").Payload, Is.EqualTo(seeded.Payload)); + Stage(); + } + + cache.BackingStores[0].PushAll(); + Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v3"), "an id added after staging marks the flush"); + + deck.Reels.Clear(); + Stage(); + cache.BackingStores[0].PushAll(); + Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v1"), "the last id taken away after staging unmarks it"); + } + + // A store with no cache has no codec to read a variant's override values with, so it cannot see whether they hold an id: a + // variant it writes counts as holding one. The same record read through a cache is marked by what its values hold. A + // flush that stages nothing copies the variant as stored. + [Test] + public void AVariantAStoreCannotReadCountsAsHoldingIdsWhenItWritesIt() + { + var path = PathOf("detached-variant.cc"); + using (var cache = Open(path)) + { + cache.Commit(batch => + { + batch.Upsert(typeof(IdDeck), EmptyDeck("base"), new CultRecordKey("base")); + batch.UpsertVariant(new CultRecordKey("v"), new CultRecordKey("base"), new[] { cache.Override(nameof(IdDeck.Name), "v") }); + }); + } + + using (var cache = Open(path)) + cache.BackingStores[0].PushAll(); + Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v2"), "read with a codec, a name override holds no id"); + + var blind = new SingleFileMessagePackBackingStore(path); + blind.UseRegistry(Registry); + blind.PullAll(); + blind.PushAll(); + Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v2"), "a flush that staged nothing copies the variant and its header"); + + blind.Push(new CultStoredDocument( + new CultRecordKey("v"), + DateTimeOffset.UtcNow.ToString("O"), + Registry.GetRequired(typeof(IdDeck)), + new CultVariantDelta("base", Array.Empty()), + null, + null)); + blind.PushAll(); + Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v3"), "read without a codec, the variant it writes cannot be seen, so it is marked"); + } + + // Through a real attach: the second cache reads the variant from the file, and the flush it then makes writes the header its + // resolved values hold, never the one a codecless copy would claim (a codecless variant counts as holding ids). + [TestCase("name", "cultcache.store.v2")] + [TestCase("reels", "cultcache.store.v3")] + public void AnAttachedVariantStoreFlushesTheHeaderItsValuesHold(string overridden, string header) + { + var path = PathOf("attached-variant-" + overridden + ".cc"); + using (var cache = Open(path)) + { + cache.Commit(batch => + { + batch.Upsert(typeof(IdDeck), EmptyDeck("base"), new CultRecordKey("base")); + var name = cache.Override(nameof(IdDeck.Name), "v"); + batch.UpsertVariant(new CultRecordKey("v"), new CultRecordKey("base"), overridden == "name" + ? new[] { name } + : new[] { name, cache.Override(nameof(IdDeck.Reels), new List { new() { Label = "r", Id = HexA } }) }); + }); + } + + using (var second = Open(path)) + { + second.Commit(batch => batch.Upsert(typeof(IdDeck), EmptyDeck("other"), new CultRecordKey("other"))); + second.FlushAllBackingStores(); + } + + Assert.That(HeaderOf(path), Is.EqualTo(header)); + } + + [Test] + public void ADirectoryStoreIsMarkedByTheIdsItWritesAndAnEmptyDeckDoesNotMarkIt() + { + var empty = PathOf("empty-dir.cc"); + using (var cache = OpenWith(empty, true, typeof(IdDeck))) + cache.Commit(batch => batch.Upsert(typeof(IdDeck), EmptyDeck("e"), new CultRecordKey("e"))); + Assert.That(HeaderOf(empty), Is.EqualTo("cultcache.store.v4.directory-content-addressed-pages")); + + using (var cache = OpenWith(empty, true, typeof(IdDeck))) + cache.Commit(batch => batch.Upsert(typeof(IdDeck), Deck("d"), new CultRecordKey("d"))); + Assert.That(HeaderOf(empty), Is.EqualTo("cultcache.store.v5.directory-content-addressed-pages")); + } + + [Test] + public void AVariantIsMarkedByItsOverrideIdsAndOtherwiseIsV2() + { + var plain = PathOf("variant-plain.cc"); + using (var cache = Open(plain)) + { + cache.Commit(batch => + { + batch.Upsert(typeof(IdDeck), EmptyDeck("base"), new CultRecordKey("base")); + batch.UpsertVariant(new CultRecordKey("v"), new CultRecordKey("base"), new[] { cache.Override(nameof(IdDeck.Name), "v") }); + }); + } + + Assert.That(HeaderOf(plain), Is.EqualTo("cultcache.store.v2")); + + var withIds = PathOf("variant-ids.cc"); + using (var cache = Open(withIds)) + { + cache.Commit(batch => + { + batch.Upsert(typeof(IdDeck), EmptyDeck("base"), new CultRecordKey("base")); + batch.UpsertVariant(new CultRecordKey("v"), new CultRecordKey("base"), + new[] { cache.Override(nameof(IdDeck.Name), "v"), cache.Override(nameof(IdDeck.Reels), new List { new() { Label = "r" } }) }); + }); + } + + Assert.That(HeaderOf(withIds), Is.EqualTo("cultcache.store.v3")); + + // Reloaded, the variant is the record the store keeps: a whole-store flush beside an empty deck still marks the file. + using (var cache = Open(withIds)) + { + cache.UpsertAsync(typeof(IdDeck), EmptyDeck("e"), new CultRecordKey("e")).GetAwaiter().GetResult(); + cache.FlushAllBackingStores(); + } + + Assert.That(HeaderOf(withIds), Is.EqualTo("cultcache.store.v3")); + + // Flattened, the variant is a plain record holding the ids its override brought; the base holds none. + using (var cache = Open(withIds)) + { + cache.FlattenAsync(new CultRecordKey("v")).GetAwaiter().GetResult(); + cache.Remove(new CultRecordKey("e")); + cache.FlushAllBackingStores(); + } + + Assert.That(HeaderOf(withIds), Is.EqualTo("cultcache.store.v3")); + } + + // Soul's probe: the base gains ids in the batch that flattens the variant, then leaves. The flattened record is the + // document it stores, so its ids mark the file whatever the base did before or after. + [TestCase(false)] + [TestCase(true)] + public void AFlattenedVariantIsMarkedByTheDocumentItStoresNotByTheOneItWasPlannedAgainst(bool viaFlush) + { + var path = PathOf(viaFlush ? "flatten-flush.cc" : "flatten-commit.cc"); + using (var cache = Open(path)) + { + cache.Commit(batch => + { + batch.Upsert(typeof(IdDeck), EmptyDeck("base"), new CultRecordKey("base")); + batch.UpsertVariant(new CultRecordKey("v"), new CultRecordKey("base"), new[] { cache.Override(nameof(IdDeck.Name), "v") }); + }); + Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v2")); + + cache.Commit(batch => + { + batch.Upsert(typeof(IdDeck), Deck("base", reels: 1), new CultRecordKey("base")); + batch.Flatten(new CultRecordKey("v")); + }); + Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v3")); + + if (viaFlush) + { + cache.Remove(new CultRecordKey("base")); + cache.FlushAllBackingStores(); + } + else + { + Assert.That(cache.Commit(batch => batch.Remove(new CultRecordKey("base"))), Is.True); + } + } + + var flattenedIds = AllIds(MessagePackSerializer.Deserialize(DiskRecord(path, "v").Payload)); + Assert.That(flattenedIds, Is.Not.Empty, "the flattened record holds the base's ids"); + Assert.That(flattenedIds, Has.All.Not.Empty); + Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v3"), "the file holds ids, so it is marked"); + } + + // A conditional commit lands onto the file as it is: it sees only its own batch, so ids only this cache minted (they are on no + // disk) do not mark the file. + [Test] + public void AConditionalCommitOntoLoadMintedIdsDoesNotMarkTheFile() + { + Type[] types = { typeof(IdDeck), typeof(PreCut2FixtureItem) }; + var preId = PathOf("conditional-minted.cc"); + WritePreIdStore(preId, null, ("a", "a", 1)); + using (var cache = OpenWith(preId, false, types)) + { + Assert.That(cache.Commit(batch => + { + batch.Expect(new CultRecordKey("plain"), null); + batch.Upsert(typeof(PreCut2FixtureItem), new PreCut2FixtureItem { Name = "plain" }, new CultRecordKey("plain")); + }), Is.True); + } + + Assert.That(HeaderOf(preId), Is.EqualTo("cultcache.store.v1"), "a's ids exist only in memory; the file holds none"); + } + + // The last id leaves the store in this commit: an unconditional commit writes the store it will hold, not the one it held. + [Test] + public void AnUnconditionalCommitThatRemovesOrReplacesTheLastIdWritesV1() + { + var removed = PathOf("last-removed.cc"); + using (var cache = Open(removed)) + { + cache.Commit(batch => batch.Upsert(typeof(IdDeck), Deck("d"), new CultRecordKey("d"))); + Assert.That(HeaderOf(removed), Is.EqualTo("cultcache.store.v3")); + cache.Commit(batch => batch.Remove(new CultRecordKey("d"))); + } + + Assert.That(HeaderOf(removed), Is.EqualTo("cultcache.store.v1")); + + var replaced = PathOf("last-replaced.cc"); + using (var cache = Open(replaced)) + { + cache.Commit(batch => batch.Upsert(typeof(IdDeck), Deck("d"), new CultRecordKey("d"))); + cache.Commit(batch => batch.Upsert(typeof(IdDeck), EmptyDeck("d"), new CultRecordKey("d"))); + } + + Assert.That(HeaderOf(replaced), Is.EqualTo("cultcache.store.v1")); + } + + // Commits one plain item through a cache over `path` and reports how many copied records that write decoded to ask whether + // they hold an element id, and the header it left. + private (int Decoded, string Header) CommitItemCountingDecodes(string path, params Type[] types) + { + var registry = CultDocumentRegistry.ForTypes(types); + var store = new SingleFileMessagePackBackingStore(path); + using var cache = new CultCache(registry, CultCacheMessagePack.CreateCodec(registry)); + cache.AddBackingStore(store); + cache.PullAllBackingStoresAsync().GetAwaiter().GetResult(); + store.CopiedRecordsDecoded = 0; + cache.Commit(batch => batch.Upsert(typeof(PreCut2FixtureItem), new PreCut2FixtureItem { Name = "p" }, new CultRecordKey("p"))); + return (store.CopiedRecordsDecoded, HeaderOf(path)); + } + + // The copy check decodes a record only where the answer can change: each gate, removed, makes one of these counts wrong. + [Test] + public void TheCopyCheckDecodesOnlyWhatCouldMarkTheStore() + { + // A v3 header is already marked: nothing to decide. + var marked = PathOf("gate-marked.cc"); + WritePreIdStore(marked, null, ("a", "a", 1), ("b", "b", 1), ("c", "c", 1)); + var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(marked)); + snapshot.FormatVersion = CultPersistedStoreSnapshot.FormatV3; + File.WriteAllBytes(marked, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + Assert.That(CommitItemCountingDecodes(marked, typeof(IdDeck), typeof(PreCut2FixtureItem)), Is.EqualTo((0, "cultcache.store.v3")), "a marked store decodes nothing"); + + // Records whose type reaches no object list can hold no id. + var noLists = PathOf("gate-no-lists.cc"); + using (var seed = OpenWith(noLists, false, typeof(PreCut2FixtureItem))) + { + foreach (var key in new[] { "x", "y", "z" }) + UpsertPlain(seed, key); + } + + Assert.That(CommitItemCountingDecodes(noLists, typeof(PreCut2FixtureItem)), Is.EqualTo((0, "cultcache.store.v1")), "a type with no object list is not decoded"); + + // A written record that holds an id already marks the store, whichever of the written records it is. + var stagedPath = PathOf("gate-staged.cc"); + WritePreIdStore(stagedPath, null, ("a", "a", 1), ("b", "b", 1), ("c", "c", 1)); + var stagedStore = new SingleFileMessagePackBackingStore(stagedPath); + var stagedRegistry = CultDocumentRegistry.ForTypes(new[] { typeof(IdDeck), typeof(PreCut2FixtureItem) }); + using (var cache = new CultCache(stagedRegistry, CultCacheMessagePack.CreateCodec(stagedRegistry))) + { + cache.AddBackingStore(stagedStore); + cache.PullAllBackingStoresAsync().GetAwaiter().GetResult(); + stagedStore.CopiedRecordsDecoded = 0; + cache.Commit(batch => + { + batch.Upsert(typeof(PreCut2FixtureItem), new PreCut2FixtureItem { Name = "plain" }, new CultRecordKey("plain")); + batch.Upsert(typeof(IdDeck), Deck("fresh"), new CultRecordKey("fresh")); + }); + } + + Assert.That(stagedStore.CopiedRecordsDecoded, Is.EqualTo(0), "a staged record holding an id marks without asking"); + Assert.That(HeaderOf(stagedPath), Is.EqualTo("cultcache.store.v3")); + + // Id-capable records without ids: each is decoded once, and the store stays unmarked. + var capable = PathOf("gate-capable.cc"); + WritePreIdStore(capable, null, ("a", "a", 1), ("b", "b", 1), ("c", "c", 1)); + Assert.That(CommitItemCountingDecodes(capable, typeof(IdDeck), typeof(PreCut2FixtureItem)), Is.EqualTo((3, "cultcache.store.v1"))); + + // A record the writer's registry does not claim is foreign: no type, no decode. + var foreign = PathOf("gate-foreign.cc"); + WritePreIdStore(foreign, null, ("a", "a", 1), ("b", "b", 1), ("c", "c", 1)); + Assert.That(CommitItemCountingDecodes(foreign, typeof(PreCut2FixtureItem)), Is.EqualTo((0, "cultcache.store.v1")), "a foreign record is not decoded"); + + // A variant is copied as its bytes: only a C# write of one holding ids marks the store, and that write marked it already. + var variant = PathOf("gate-variant.cc"); + WriteV2StoreWithAnIdLessVariant(variant); + Assert.That(CommitItemCountingDecodes(variant, typeof(IdDeck), typeof(PreCut2FixtureItem)), Is.EqualTo((1, "cultcache.store.v2")), "the plain base is decoded, the variant is not"); + } + + // Soul's probe: a C1 v2 store whose variant override holds no ids. Loading mints them in memory; the store keeps the delta + // it read, so the file holds none and no write of a plain record beside it may mark the file. + private static void WriteV2StoreWithAnIdLessVariant(string path) + { + using (var seed = OpenWith(path, false, typeof(IdDeck), typeof(PreCut2FixtureItem))) + { + seed.Commit(batch => + { + batch.Upsert(typeof(IdDeck), EmptyDeck("base"), new CultRecordKey("base")); + batch.UpsertVariant(new CultRecordKey("v"), new CultRecordKey("base"), + new[] { seed.Override(nameof(IdDeck.Name), "v"), seed.Override(nameof(IdDeck.Reels), new List { new() }) }); + }); + } + + var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + var record = snapshot.Records.Single(entry => entry.Key == "v"); + record.Variant = new CultVariantDelta(record.Variant!.BaseKey, record.Variant.Overrides + .Select(entry => entry.Path[0].Slot == 1 ? CultVariantOverride.Set(1, OldShapedReels("old")) : entry).ToArray()); + snapshot.FormatVersion = CultPersistedStoreSnapshot.FormatV2; + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + } + + [TestCase("unconditional")] + [TestCase("conditional")] + [TestCase("flush")] + public void AVariantWhoseIdsExistOnlyInMemoryDoesNotMarkTheFileAPlainWriteLandsIn(string write) + { + var path = PathOf("variant-in-memory-" + write + ".cc"); + WriteV2StoreWithAnIdLessVariant(path); + Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v2")); + + using (var cache = OpenWith(path, false, typeof(IdDeck), typeof(PreCut2FixtureItem))) + { + Assert.That(AllIds(cache.Get(new CultRecordKey("v"))!), Has.All.Not.Empty, "the variant holds minted ids in memory"); + var key = new CultRecordKey("p"); + switch (write) + { + case "unconditional": + UpsertPlain(cache, "p"); + break; + case "conditional": + Assert.That(cache.Commit(batch => + { + batch.Expect(key, null); + batch.Upsert(typeof(PreCut2FixtureItem), new PreCut2FixtureItem { Name = "p" }, key); + }), Is.True); + break; + default: + cache.UpsertAsync(typeof(PreCut2FixtureItem), new PreCut2FixtureItem { Name = "p" }, key).GetAwaiter().GetResult(); + cache.FlushAllBackingStores(); + break; + } + } + + Assert.That(DiskRecord(path, "p"), Is.Not.Null); + Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v2"), "the file holds no id, whatever this cache minted in memory"); + } + + // Soul's A/B probe: a conditional commit lands onto the file as it is, so a stale view of what the file once held does not + // decide the header. A holds a deck with ids; B removes it; A commits a plain record conditionally. + [Test] + public void AConditionalCommitIsNotMarkedByIdsItsStaleViewHoldsAndTheFileNoLongerDoes() + { + var path = PathOf("stale-view.cc"); + Type[] types = { typeof(IdDeck), typeof(PreCut2FixtureItem) }; + using var a = OpenWith(path, false, types); + a.Commit(batch => batch.Upsert(typeof(IdDeck), Deck("d"), new CultRecordKey("d"))); + using (var b = OpenWith(path, false, types)) + b.Commit(batch => batch.Remove(new CultRecordKey("d"))); + Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v1"), "B's whole-store commit left no id"); + Assert.That(a.Get(new CultRecordKey("d")), Is.Not.Null, "A has not pulled: its view is stale"); + + Assert.That(a.Commit(batch => + { + batch.Expect(new CultRecordKey("plain"), null); + batch.Upsert(typeof(PreCut2FixtureItem), new PreCut2FixtureItem { Name = "plain" }, new CultRecordKey("plain")); + }), Is.True); + + Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v1"), "the merged file holds no id"); + } + + // Any override holding an id marks the variant, wherever it sits among the overrides. + [Test] + public void AVariantWhoseIdBearingOverrideIsNotTheLastIsStillMarked() + { + var path = PathOf("variant-order.cc"); + using (var cache = Open(path)) + { + cache.Commit(batch => + { + batch.Upsert(typeof(IdDeck), EmptyDeck("base"), new CultRecordKey("base")); + batch.UpsertVariant(new CultRecordKey("v"), new CultRecordKey("base"), + new[] { cache.Override(nameof(IdDeck.Reels), new List { new() { Label = "r" } }), cache.Override(nameof(IdDeck.Name), "v") }); + }); + } + + Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v3")); + } + + [Test] + public void AnElementObjectTwiceInOneListIsRefusedAtWriteAndNothingIsPersisted() + { + var path = PathOf("shared.cc"); + using (var cache = Open(path)) + { + var shared = new IdMark { Text = "s" }; + var deck = new IdDeck { Name = "shared", Reels = { new IdReel { Label = "r", Marks = { shared, shared } } } }; + var refusal = Assert.Throws(() => cache.Commit(batch => batch.Upsert(typeof(IdDeck), deck, new CultRecordKey("shared"))))!; + Assert.That(refusal.Message, Does.Contain("shared").And.Contain("twice")); + Assert.That(refusal.RecordKey, Is.EqualTo("shared")); + Assert.That(shared.Id, Is.Empty, "the refusal minted nothing"); + Assert.That(cache.Get(new CultRecordKey("shared")), Is.Null); + } + + Assert.That(File.Exists(path), Is.False, "nothing was persisted"); + } + + [Test] + public void ADerivedIdWhoseSourceIsEmptyIsRefusedAtWriteNamingTheSourceMember() + { + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(LabelDoc) }); + using var cache = CultCacheMessagePack.Create(PathOf("label.cc"), new CultCacheOpenOptions { Registry = registry }); + var doc = new LabelDoc { Name = "l", Labels = { new IdLabel { Label = "ok" }, new IdLabel { Label = null } } }; + var refusal = Assert.Throws(() => cache.Commit(batch => batch.Upsert(typeof(LabelDoc), doc, new CultRecordKey("l"))))!; + Assert.That(refusal.Member, Is.EqualTo("Label")); + Assert.That(refusal.RecordKey, Is.EqualTo("l")); + Assert.That(cache.Get(new CultRecordKey("l")), Is.Null); + } + + [CultDocument("tests.element_id_label_doc", "tests.element_id_label_doc.v1")] + [MessagePackObject] + public sealed class LabelDoc + { + [Key(0)] [CultName] public string Name { get; set; } = ""; + [Key(1)] public List Labels { get; set; } = new(); + } + + [MessagePackObject] + public sealed class IdLabel + { + [Key(0)] public string? Label { get; set; } + [Key(1)] [CultElementId(nameof(Label))] public string Id { get; set; } = ""; + } + + [CultDocument("tests.element_id_span_doc", "tests.element_id_span_doc.v1")] + [MessagePackObject] + public sealed class SpanDoc + { + [Key(0)] [CultName] public string Name { get; set; } = ""; + [Key(1)] public List Spans { get; set; } = new(); + } + + [MessagePackObject] + public sealed class IdSpan + { + [Key(0)] public long Offset { get; set; } + [Key(1)] public int Size { get; set; } + [Key(2)] [CultElementId(nameof(Offset))] public string Id { get; set; } = ""; + } + + [MessagePackObject] + public sealed class IdMark + { + [Key(0)] public string Text { get; set; } = ""; + [Key(1)] [CultElementId] public string Id { get; set; } = ""; + } + + [MessagePackObject] + public sealed class IdReel + { + [Key(0)] public string Label { get; set; } = ""; + [Key(1)] public List Marks { get; set; } = new(); + [Key(2)] [CultElementId] public string Id { get; set; } = ""; + } + + [MessagePackObject] + [Union(0, typeof(IdSlash))] + [Union(1, typeof(IdStab))] + public abstract class IdMove + { + [Key(0)] [CultElementId] public string Id { get; set; } = ""; + } + + [MessagePackObject] + public sealed class IdSlash : IdMove + { + [Key(1)] public int Reach { get; set; } + } + + [MessagePackObject] + public sealed class IdStab : IdMove + { + [Key(1)] public int Depth { get; set; } + } + + [CultDocument("tests.element_id_whole_holder", "tests.element_id_whole_holder.v1")] + [MessagePackObject] + public sealed class IdWholeHolder + { + [Key(0)] [CultName] public string Name { get; set; } = ""; + [Key(1)] public ImmutableArray Reels { get; set; } = ImmutableArray.Empty; + } + + [CultDocument("tests.element_id_deck", "tests.element_id_deck.v1")] + [MessagePackObject] + public sealed class IdDeck + { + [Key(0)] [CultName] public string Name { get; set; } = ""; + [Key(1)] public List Reels { get; set; } = new(); + [Key(2)] public List Moves { get; set; } = new(); + } + + // The shapes before ids: same slots, no id member. Not documents, so never registered. + [MessagePackObject] + public sealed class OldMark + { + [Key(0)] public string Text { get; set; } = ""; + } + + [MessagePackObject] + public sealed class OldReel + { + [Key(0)] public string Label { get; set; } = ""; + [Key(1)] public List Marks { get; set; } = new(); + } + + [MessagePackObject] + public sealed class OldDeck + { + [Key(0)] public string Name { get; set; } = ""; + [Key(1)] public List Reels { get; set; } = new(); + } + } +} diff --git a/tests/GameCult.Caching.Tests/EmittedDocumentTypes.cs b/tests/GameCult.Caching.Tests/EmittedDocumentTypes.cs new file mode 100644 index 00000000..ac6a6dad --- /dev/null +++ b/tests/GameCult.Caching.Tests/EmittedDocumentTypes.cs @@ -0,0 +1,56 @@ +#nullable enable +using System; +using System.Linq; +using System.Reflection; +using System.Reflection.Emit; +using MessagePack; + +namespace GameCult.Caching.Tests +{ + // Document types built at run time, each in its own dynamic assembly, which CultDocumentRegistry.Shared never scans. A + // fixture emits a type here when declaring it would change what Shared holds: a pair Shared refuses (one schema declared + // two ways), an alias whose resolved type would depend on scan order, or a type a test needs to itself. + internal static class EmittedDocumentTypes + { + internal readonly record struct Field(string Name, Type Type, int Key, bool IsName = false); + + internal static Type Emit(string typeName, string schemaName, string schemaVersion, Field[]? fields = null, string[]? compatibleSchemaIds = null) + { + var assembly = AssemblyBuilder.DefineDynamicAssembly( + new AssemblyName("GameCult.Caching.EmittedDocuments." + Guid.NewGuid().ToString("N")), + AssemblyBuilderAccess.Run); + var type = assembly.DefineDynamicModule("main").DefineType(typeName, TypeAttributes.Class | TypeAttributes.Public | TypeAttributes.Sealed); + var documentConstructor = typeof(CultDocumentAttribute).GetConstructor(new[] { typeof(string), typeof(string) })!; + type.SetCustomAttribute(compatibleSchemaIds == null + ? new CustomAttributeBuilder(documentConstructor, new object[] { schemaName, schemaVersion }) + : new CustomAttributeBuilder( + documentConstructor, + new object[] { schemaName, schemaVersion }, + new[] { typeof(CultDocumentAttribute).GetProperty(nameof(CultDocumentAttribute.CompatibleSchemaIds))! }, + new object[] { compatibleSchemaIds })); + type.SetCustomAttribute(new CustomAttributeBuilder( + typeof(MessagePackObjectAttribute).GetConstructor(new[] { typeof(bool) })!, + new object[] { false })); + foreach (var field in fields ?? Array.Empty()) + { + var builder = type.DefineField(field.Name, field.Type, FieldAttributes.Public); + builder.SetCustomAttribute(new CustomAttributeBuilder(typeof(KeyAttribute).GetConstructor(new[] { typeof(int) })!, new object[] { field.Key })); + if (field.IsName) + builder.SetCustomAttribute(new CustomAttributeBuilder(typeof(CultNameAttribute).GetConstructor(Type.EmptyTypes)!, Array.Empty())); + } + + return type.CreateType()!; + } + + // An instance of an emitted type with the named fields set. + internal static object New(Type type, params (string Field, object? Value)[] values) + { + var document = Activator.CreateInstance(type)!; + foreach (var (field, value) in values) + type.GetField(field)!.SetValue(document, value); + return document; + } + + internal static object? Read(object document, string field) => document.GetType().GetField(field)!.GetValue(document); + } +} diff --git a/tests/GameCult.Caching.Tests/ForeignRecordTests.cs b/tests/GameCult.Caching.Tests/ForeignRecordTests.cs new file mode 100644 index 00000000..ab4cb046 --- /dev/null +++ b/tests/GameCult.Caching.Tests/ForeignRecordTests.cs @@ -0,0 +1,1631 @@ +#nullable enable +using System; +using System.IO; +using System.Linq; +using GameCult.Caching.MessagePack; +using NUnit.Framework; +using static GameCult.Caching.Tests.EmittedDocumentTypes; + +namespace GameCult.Caching.Tests +{ + // A record under a schema id no registered type owns or lists as compatible is foreign. CultCache never destroys or relabels + // it: both store kinds carry it byte for byte, list it as foreign, and refuse any write that would replace or remove it. + public class ForeignRecordTests + { + private static readonly Type Deck = Emit("ForeignSuiteDeck", "tests.foreign_deck", "tests.foreign_deck.v1", new[] { new Field("Name", typeof(string), 0, IsName: true) }); + // The deck's schema declaring another runtime's id compatible (OtherRuntimeId below): one under the deck's own version, and a + // later version that a registry holding the deck can register beside it. + private static readonly Type ClaimingDeck = Emit("ForeignSuiteClaimingDeck", "tests.foreign_deck", "tests.foreign_deck.v1", new[] { new Field("Name", typeof(string), 0, IsName: true) }, new[] { "other.runtime.deck" }); + private static readonly Type LateClaimingDeck = Emit("ForeignSuiteLateClaimingDeck", "tests.foreign_deck", "tests.foreign_deck.v2", new[] { new Field("Name", typeof(string), 0, IsName: true) }, new[] { "other.runtime.deck" }); + private static readonly Type Widget = Emit("ForeignSuiteWidget", "tests.foreign_widget", "tests.foreign_widget.v1", new[] { new Field("Label", typeof(string), 0, IsName: true) }); + // A second type a cache registers, so that a second backing store has a home of its own. + private static readonly Type Gizmo = Emit("ForeignSuiteGizmo", "tests.foreign_gizmo", "tests.foreign_gizmo.v1", new[] { new Field("Tag", typeof(string), 0, IsName: true) }); + + // A build that has both types, and one that has only the deck: to it the widget is foreign. + private static readonly CultDocumentRegistry Full = CultDocumentRegistry.ForTypes(new[] { Deck, Widget }); + private static readonly CultDocumentRegistry DeckOnly = CultDocumentRegistry.ForTypes(new[] { Deck }); + private static readonly string WidgetId = Full.GetRequired(Widget).SchemaId; + + // Another writer's deck (an older or newer build, another runtime) under an id the deck does not declare. It declares the + // deck's own id compatible, so it may rewrite the deck's records; the deck still reads them through that listing, but only by + // dropping the Note slot it lacks, so it does not own them and holds them read-only. + private static readonly Type MovingDeck = Emit("ForeignSuiteMovingDeck", "tests.foreign_deck", "tests.foreign_deck.v3", new[] { new Field("Name", typeof(string), 0, IsName: true), new Field("Note", typeof(string), 1) }, new[] { DeckOnly.GetRequired(Deck).SchemaId }); + private static readonly CultDocumentRegistry Moving = CultDocumentRegistry.ForTypes(new[] { MovingDeck }); + private static readonly string MovingId = Moving.GetRequired(MovingDeck).SchemaId; + + private static readonly CultRecordKey D = new("d"); + private static readonly CultRecordKey E = new("e"); + private static readonly CultRecordKey W = new("w"); + private static readonly CultRecordKey V = new("v"); + private const string Canary = "CANARY-7f3c-payload-value"; + + public enum Write + { + Flush, + Commit, + ConditionalCommit + } + + private string _directory = ""; + + [SetUp] + public void SetUp() + { + _directory = Path.Combine(Path.GetTempPath(), "cultlib-foreign-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(_directory); + } + + [TearDown] + public void TearDown() + { + if (Directory.Exists(_directory)) + Directory.Delete(_directory, true); + } + + private string PathOf(string name) => Path.Combine(_directory, name); + + private static CultCache Open(string path, CultDocumentRegistry registry, bool directory) => + CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = registry, UseDirectoryStore = directory }); + + private static object DeckOf(string name) => New(Deck, ("Name", name)); + + private static object WidgetOf(string label) => New(Widget, ("Label", label)); + + private static CultPersistedStoreSnapshot Read(string path) => + CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + + // Every file of the store, by name, with its bytes: a refused write leaves all of them as they were. + private static string[] Fingerprint(string path) + { + var files = new[] { path }.Concat(Directory.Exists(DirectoryMessagePackBackingStore.DefaultRecordDirectoryPath(path)) + ? Directory.GetFiles(DirectoryMessagePackBackingStore.DefaultRecordDirectoryPath(path), "*.msgpack") + : Array.Empty()) + .OrderBy(file => file, StringComparer.Ordinal); + return files.Select(file => Path.GetFileName(file) + ":" + Convert.ToBase64String(File.ReadAllBytes(file))).ToArray(); + } + + // What the store holds for one record as the file (single-file) or manifest (directory: the page hash) names it. + private static string Stored(string path, CultRecordKey key) + { + var record = Read(path).Records.Single(candidate => candidate.Key == key.Value); + return $"{record.SchemaId}|{record.StoredAt}|{Convert.ToBase64String(record.Payload)}|{record.Variant == null}"; + } + + // A store holding a deck d and a widget w, written by the build that has both types. + private string Seeded(string name, bool directory) + { + var path = PathOf(name); + using var cache = Open(path, Full, directory); + cache.Commit(batch => + { + batch.Upsert(Deck, DeckOf("d"), D); + batch.Upsert(Widget, WidgetOf("w"), W); + }); + return path; + } + + private static CacheBackingStore StoreOf(CultCache cache) => cache.BackingStores.Single(); + + private static void Land(CultCache cache, Write write, CultRecordKey key, object document) + { + switch (write) + { + case Write.Flush: + cache.UpsertAsync(document.GetType(), document, key).GetAwaiter().GetResult(); + cache.FlushAsync().GetAwaiter().GetResult(); + break; + case Write.Commit: + Assert.That(cache.Commit(batch => batch.Upsert(document.GetType(), document, key)), Is.True); + break; + case Write.ConditionalCommit: + Assert.That(cache.Commit(batch => + { + batch.Expect(D, cache.Get(D)); + batch.Upsert(document.GetType(), document, key); + }), Is.True); + break; + } + } + + private static void Refused(CultCache cache, Write write, CultRecordKey key, object document, CultRecordKey foreignKey) + { + var refusal = Assert.Throws(() => Land(cache, write, key, document))!; + Assert.Multiple(() => + { + Assert.That(refusal.RecordKey, Is.EqualTo(foreignKey.Value)); + Assert.That(refusal.SchemaId, Is.EqualTo(WidgetId)); + Assert.That(refusal.SchemaNames, Is.EqualTo(new[] { "tests.foreign_widget" })); + Assert.That(refusal.Message, Does.Contain(foreignKey.Value).And.Contain(WidgetId)); + Assert.That(refusal.Message, Does.Not.Contain(Canary), "a refusal never echoes the document it refused"); + }); + } + + [TestCase(false)] + [TestCase(true)] + public void AForeignRecordLoadsAsForeignAndTheCacheDoesNotHoldIt(bool directory) + { + var path = Seeded("seeded.cc", directory); + var storedAt = Read(path).Records.Single(record => record.Key == W.Value).StoredAt; + + using var cache = Open(path, DeckOnly, directory); + + var foreign = StoreOf(cache).ForeignRecords.Single(); + Assert.Multiple(() => + { + Assert.That(foreign.Key, Is.EqualTo(W.Value)); + Assert.That(foreign.SchemaId, Is.EqualTo(WidgetId)); + Assert.That(foreign.SchemaName, Is.EqualTo("tests.foreign_widget")); + Assert.That(foreign.StoredAt, Is.EqualTo(storedAt)); + Assert.That(cache.Get(W), Is.Null, "the cache never holds a foreign record"); + Assert.That(EmittedDocumentTypes.Read(cache.Get(D)!, "Name"), Is.EqualTo("d"), "the records it claims load as usual"); + }); + } + + // Every write path lays the foreign record back as the file holds it: the single-file store's whole view (a flush and an + // unconditional commit), its commit onto the file, and the directory store's generation. + [Test] + public void AWriteOfAnotherRecordKeepsTheForeignRecordByteForByte( + [Values(false, true)] bool directory, + [Values] Write write) + { + var path = Seeded("kept.cc", directory); + var before = Stored(path, W); + + using (var cache = Open(path, DeckOnly, directory)) + Land(cache, write, E, DeckOf("e")); + + Assert.That(Stored(path, W), Is.EqualTo(before)); + Assert.That(Read(path).Records.Select(record => record.Key), Is.EquivalentTo(new[] { "d", "e", "w" })); + using var full = Open(path, Full, directory); + Assert.That(EmittedDocumentTypes.Read(full.Get(W)!, "Label"), Is.EqualTo("w"), "the build that has the type still reads it"); + Assert.That(StoreOf(full).ForeignRecords, Is.Empty); + } + + [Test] + public void AWriteThatWouldReplaceAForeignRecordIsRefusedAndNothingIsWritten( + [Values(false, true)] bool directory, + [Values] Write write) + { + var path = Seeded("replace.cc", directory); + var before = Fingerprint(path); + + using var cache = Open(path, DeckOnly, directory); + Refused(cache, write, W, DeckOf(Canary), W); + + Assert.That(Fingerprint(path), Is.EqualTo(before)); + Assert.That(cache.Get(W), Is.Null); + } + + // Another writer replaced a record this cache holds with a record of a type it does not have. A write that would lay this + // cache's record over it is refused; the file keeps the other writer's record. The refusal reloads it as a pull would: the + // store lists it foreign, so the next write of it is refused before staging. + [Test] + public void AWriteOverARecordAnotherWriterMadeForeignIsRefused([Values] bool directory, [Values(Write.Flush, Write.Commit)] Write write) + { + var path = Seeded("replaced.cc", directory); + using var cache = Open(path, DeckOnly, directory); + using (var other = Open(path, Full, directory)) + other.Commit(batch => batch.Upsert(Widget, WidgetOf("now a widget"), D)); + var before = Fingerprint(path); + + var refusal = Assert.Throws(() => Land(cache, write, D, DeckOf(Canary)))!; + + Assert.That(refusal.RecordKey, Is.EqualTo(D.Value)); + Assert.That(refusal.Message, Does.Not.Contain(Canary)); + Assert.That(Fingerprint(path), Is.EqualTo(before)); + Assert.That(StoreOf(cache).ForeignRecords.Select(record => record.Key), Is.EqualTo(new[] { "d", "w" })); + Assert.Throws(() => cache.UpsertAsync(Deck, DeckOf(Canary), D).GetAwaiter().GetResult()); + Assert.That(cache.IsDirty, Is.False); + } + + // A refusal does not poison the store: what the cache held for the refused record goes with the refusal, so a later write + // to another record lands and the file keeps the other writer's record. + [Test] + public void AfterARefusedWriteAWriteToAnotherRecordLands( + [Values(false, true)] bool directory, + [Values(Write.Flush, Write.Commit)] Write refused) + { + var path = Seeded("unstuck.cc", directory); + using var cache = Open(path, DeckOnly, directory); + using (var other = Open(path, Full, directory)) + other.Commit(batch => batch.Upsert(Widget, WidgetOf("now a widget"), D)); + var widget = Stored(path, D); + + Assert.Throws(() => Land(cache, refused, D, DeckOf(Canary))); + Land(cache, Write.Commit, E, DeckOf("e")); + + Assert.That(Stored(path, D), Is.EqualTo(widget), "the other writer's record is as it was"); + Assert.That(EmittedDocumentTypes.Read(cache.Get(E)!, "Name"), Is.EqualTo("e")); + Assert.That(Read(path).Records.Select(record => record.Key), Does.Contain("e")); + } + + // The cache and the store agree on what a refusal left: the refused record is not served, not after a later write, not after + // a pull, and not by a fresh open. What the cache served before the refusal was never what the file holds. + [Test] + public void ARefusedRecordIsNotServedAfterTheRefusalAfterALaterWriteOrAfterAPull( + [Values(false, true)] bool directory, + [Values(Write.Flush, Write.Commit)] Write refused) + { + var path = Seeded("not-served.cc", directory); + using var cache = Open(path, DeckOnly, directory); + using (var other = Open(path, Full, directory)) + other.Commit(batch => batch.Upsert(Widget, WidgetOf("now a widget"), D)); + + var refusal = Assert.Throws(() => Land(cache, refused, D, DeckOf(Canary)))!; + Assert.That(refusal.Message, Does.StartWith("Record 'd' (schema id '" + WidgetId + "'"), "one refused key reads as one record"); + var afterRefusal = cache.Get(D); + Land(cache, Write.Commit, E, DeckOf("e")); + var afterLaterWrite = cache.Get(D); + cache.PullAllBackingStoresAsync().GetAwaiter().GetResult(); + var afterPull = cache.Get(D); + using var fresh = Open(path, DeckOnly, directory); + + Assert.Multiple(() => + { + Assert.That(afterRefusal, Is.Null, "after the refusal"); + Assert.That(afterLaterWrite, Is.Null, "after a later write"); + Assert.That(afterPull, Is.Null, "after a pull"); + Assert.That(fresh.Get(D), Is.Null, "a fresh open"); + Assert.That(StoreOf(cache).ForeignRecords.Select(record => record.Key), Is.EqualTo(StoreOf(fresh).ForeignRecords.Select(record => record.Key))); + }); + } + + // A refusal names every key it refused, and every one of them leaves the cache; a record the write did not touch stays. + [Test] + public void ARefusalOfSeveralRecordsNamesEachAndTheCacheDropsEach( + [Values(false, true)] bool directory, + [Values(Write.Flush, Write.Commit)] Write refused) + { + var path = PathOf("several.cc"); + var d2 = new CultRecordKey("d2"); + using (var seed = Open(path, Full, directory)) + seed.Commit(batch => + { + batch.Upsert(Deck, DeckOf("d"), D); + batch.Upsert(Deck, DeckOf("d2"), d2); + batch.Upsert(Deck, DeckOf("e"), E); + }); + using var cache = Open(path, DeckOnly, directory); + using (var other = Open(path, Full, directory)) + other.Commit(batch => + { + batch.Upsert(Widget, WidgetOf("w1"), D); + batch.Upsert(Widget, WidgetOf("w2"), d2); + }); + + var refusal = Assert.Throws(() => + { + if (refused == Write.Flush) + { + cache.UpsertAsync(Deck, DeckOf(Canary), D).GetAwaiter().GetResult(); + cache.UpsertAsync(Deck, DeckOf(Canary), d2).GetAwaiter().GetResult(); + cache.FlushAsync().GetAwaiter().GetResult(); + } + else + { + cache.Commit(batch => + { + batch.Upsert(Deck, DeckOf(Canary), D); + batch.Upsert(Deck, DeckOf(Canary), d2); + }); + } + })!; + cache.PullAllBackingStoresAsync().GetAwaiter().GetResult(); + + Assert.Multiple(() => + { + Assert.That(refusal.RecordKey, Is.EqualTo("d"), "the first refused key in key order"); + Assert.That(refusal.Message, Does.Contain("Records 'd' (schema id '" + WidgetId + "'").And.Contain("'d2' (schema id '" + WidgetId + "'").And.Not.Contain(Canary)); + Assert.That(cache.Get(D), Is.Null); + Assert.That(cache.Get(d2), Is.Null); + Assert.That(EmittedDocumentTypes.Read(cache.Get(E)!, "Name"), Is.EqualTo("e"), "a record the write did not touch stays"); + }); + } + + // A write that never touches a record another writer made foreign is not refused for holding a stale copy of it: it lands + // the first time, in both store kinds, and the next pull drops the copy. + [Test] + public void AWriteThatNeverTouchedARecordAnotherWriterMadeForeignLandsAtOnce( + [Values(false, true)] bool directory, + [Values(Write.Flush, Write.Commit)] Write write) + { + var path = Seeded("untouched.cc", directory); + using var cache = Open(path, DeckOnly, directory); + using (var other = Open(path, Full, directory)) + other.Commit(batch => batch.Upsert(Widget, WidgetOf("now a widget"), D)); + var widget = Stored(path, D); + + Land(cache, write, E, DeckOf("e")); + cache.PullAllBackingStoresAsync().GetAwaiter().GetResult(); + + Assert.Multiple(() => + { + Assert.That(Stored(path, D), Is.EqualTo(widget), "the other writer's record is as it was"); + Assert.That(Read(path).Records.Select(record => record.Key), Does.Contain("e")); + Assert.That(cache.Get(D), Is.Null, "the pull drops the stale copy"); + }); + } + + // The write carried the file's record for d; it did not store this cache's copy, so the copy's stamp is still what the + // cache read, and a condition on it still fails against the other writer's record. + [Test] + public void ACarriedForeignRecordDoesNotRestampTheStaleCopyTheCacheHolds( + [Values(false, true)] bool directory, + [Values(Write.Flush, Write.Commit)] Write write) + { + var path = Seeded("restamp.cc", directory); + using var cache = Open(path, DeckOnly, directory); + using (var other = Open(path, Full, directory)) + other.Commit(batch => batch.Upsert(Widget, WidgetOf("now a widget"), D)); + + Land(cache, write, E, DeckOf("e")); + + Assert.That(cache.Commit(batch => + { + batch.Expect(D, cache.Get(D)); + batch.Upsert(Deck, DeckOf("e2"), E); + }), Is.False, "the cache never read the other writer's record"); + } + + // The same when the stale copy was only held, never written: a flush after the other writer's record landed refuses once. + [Test] + public void AFlushRefusedForAStaleCopyOfAForeignRecordRefusesOnce([Values(false, true)] bool directory) + { + var path = Seeded("stale.cc", directory); + using var cache = Open(path, DeckOnly, directory); + using (var other = Open(path, Full, directory)) + other.Commit(batch => batch.Upsert(Widget, WidgetOf("now a widget"), D)); + var widget = Stored(path, D); + + Assert.Throws(() => Land(cache, Write.Commit, D, DeckOf(Canary))); + Land(cache, Write.Flush, E, DeckOf("e")); + Land(cache, Write.Flush, E, DeckOf("e again")); + + Assert.That(Stored(path, D), Is.EqualTo(widget)); + } + + // A key staged and not yet written is refused by any write that carries it, whatever key that write names: an unconditional + // commit writes what a flush would. + [Test] + public void AStagedWriteIsRefusedByACommitOfAnotherRecord([Values(false, true)] bool directory) + { + var path = Seeded("staged-commit.cc", directory); + using var cache = Open(path, DeckOnly, directory); + using (var other = Open(path, Full, directory)) + other.Commit(batch => batch.Upsert(Widget, WidgetOf("now a widget"), D)); + var widget = Stored(path, D); + + cache.UpsertAsync(Deck, DeckOf(Canary), D).GetAwaiter().GetResult(); + var refusal = Assert.Throws(() => cache.Commit(batch => batch.Upsert(Deck, DeckOf("e"), E)))!; + + Assert.That(refusal.RecordKey, Is.EqualTo(D.Value)); + Assert.That(Stored(path, D), Is.EqualTo(widget)); + } + + // What a write landed is no longer staged: a record that becomes foreign afterwards is the other writer's, and writing + // another record is not refused for it. + [Test] + public void AWrittenRecordIsNoLongerStagedSoALaterForeignRecordDoesNotRefuseOtherWrites( + [Values(false, true)] bool directory, + [Values(Write.Flush, Write.Commit)] Write landed) + { + var path = Seeded("landed.cc", directory); + using var cache = Open(path, DeckOnly, directory); + Land(cache, landed, D, DeckOf("d again")); + using (var other = Open(path, Full, directory)) + other.Commit(batch => batch.Upsert(Widget, WidgetOf("now a widget"), D)); + var widget = Stored(path, D); + + Land(cache, Write.Commit, E, DeckOf("e")); + + Assert.That(Stored(path, D), Is.EqualTo(widget)); + Assert.That(Read(path).Records.Select(record => record.Key), Does.Contain("e")); + } + + // A commit writes the whole view, so what it landed that was staged is no longer staged either. + [Test] + public void AStagedWriteLandedByACommitIsNoLongerStaged([Values(false, true)] bool directory) + { + var path = Seeded("staged-landed.cc", directory); + using var cache = Open(path, DeckOnly, directory); + cache.UpsertAsync(Deck, DeckOf("d staged"), D).GetAwaiter().GetResult(); + Land(cache, Write.Commit, E, DeckOf("e")); + using (var other = Open(path, Full, directory)) + other.Commit(batch => batch.Upsert(Widget, WidgetOf("now a widget"), D)); + var widget = Stored(path, D); + + Land(cache, Write.Commit, new CultRecordKey("e2"), DeckOf("e2")); + + Assert.That(Stored(path, D), Is.EqualTo(widget)); + Assert.That(Read(path).Records.Select(record => record.Key), Does.Contain("e2")); + } + + // A staged removal is a write of its own: a refusal takes it too. + [TestCase(false)] + [TestCase(true)] + public void AStagedRemovalOfARecordAnotherWriterMadeForeignIsRefusedOnceAndThenGone(bool directory) + { + var path = Seeded("staged-removal.cc", directory); + using var cache = Open(path, DeckOnly, directory); + using (var other = Open(path, Full, directory)) + other.Commit(batch => batch.Upsert(Widget, WidgetOf("now a widget"), D)); + var widget = Stored(path, D); + + cache.Remove(D); + var refusal = Assert.Throws(() => cache.FlushAsync().GetAwaiter().GetResult())!; + Land(cache, Write.Commit, E, DeckOf("e")); + + Assert.That(refusal.RecordKey, Is.EqualTo(D.Value)); + Assert.That(Stored(path, D), Is.EqualTo(widget)); + } + + // What a refusal leaves staged is exactly what it did not refuse: the other record's write still lands on the next flush, + // and a store with nothing left staged is clean. + [TestCase(false)] + [TestCase(true)] + public void ARefusalLeavesStagedWhatItDidNotRefuse(bool directory) + { + var path = Seeded("pending.cc", directory); + using var cache = Open(path, DeckOnly, directory); + using (var other = Open(path, Full, directory)) + other.Commit(batch => batch.Upsert(Widget, WidgetOf("now a widget"), D)); + + cache.UpsertAsync(Deck, DeckOf(Canary), D).GetAwaiter().GetResult(); + Assert.Throws(() => cache.FlushAsync().GetAwaiter().GetResult()); + Assert.That(cache.IsDirty, Is.False, "nothing is left staged"); + + cache.Commit(batch => batch.Upsert(Deck, DeckOf("e"), E)); + cache.UpsertAsync(Deck, DeckOf(Canary), E).GetAwaiter().GetResult(); + cache.UpsertAsync(Deck, DeckOf("v"), V).GetAwaiter().GetResult(); + using (var other = Open(path, Full, directory)) + other.Commit(batch => batch.Upsert(Widget, WidgetOf("now a widget"), E)); + Assert.Throws(() => cache.FlushAsync().GetAwaiter().GetResult()); + Assert.That(cache.IsDirty, Is.True, "v is still staged"); + cache.FlushAsync().GetAwaiter().GetResult(); + + Assert.That(Read(path).Records.Select(record => record.Key), Does.Contain("v")); + Assert.That(cache.IsDirty, Is.False); + } + + [TestCase(false)] + [TestCase(true)] + public void RemovingARecordAnotherWriterMadeForeignIsRefused(bool directory) + { + var path = Seeded("removed.cc", directory); + using var cache = Open(path, DeckOnly, directory); + cache.Commit(batch => batch.Upsert(Deck, DeckOf("e"), E)); + using (var other = Open(path, Full, directory)) + other.Commit(batch => batch.Upsert(Widget, WidgetOf("now a widget"), D)); + var before = Fingerprint(path); + + // A condition keeps the single-file commit on the file as it is, so only the removal reaches d. + var refusal = Assert.Throws(() => cache.Commit(batch => + { + batch.Expect(E, cache.Get(E)); + batch.Remove(D); + }))!; + + Assert.That(refusal.RecordKey, Is.EqualTo(D.Value)); + Assert.That(Fingerprint(path), Is.EqualTo(before)); + } + + [TestCase(false)] + [TestCase(true)] + public void ARecordAnotherWriterMadeForeignLeavesTheCacheAtThePull(bool directory) + { + var path = Seeded("pulled.cc", directory); + using var cache = Open(path, DeckOnly, directory); + using (var other = Open(path, Full, directory)) + other.Commit(batch => batch.Upsert(Widget, WidgetOf("now a widget"), D)); + + cache.PullAllBackingStoresAsync().GetAwaiter().GetResult(); + + Assert.That(cache.Get(D), Is.Null); + Assert.That(StoreOf(cache).ForeignRecords.Select(record => record.Key), Is.EqualTo(new[] { "d", "w" })); + } + + // A directory load without a lease that a writer's commit forces to retry lists each foreign record once, from the manifest + // the retry read. + [Test] + public void ARetriedDirectoryLoadListsEachForeignRecordOnce() + { + var path = Seeded("retried.cc", directory: true); + var store = new DirectoryMessagePackBackingStore(path); + using var cache = new CultCache(DeckOnly); + cache.AddBackingStore(store); + File.Delete(Path.Combine(DirectoryMessagePackBackingStore.DefaultRecordDirectoryPath(path), ".commit.lock")); + var commits = 0; + // The writer is another cache, so it commits from another thread; it commits once, under the first unleased read. + store.UnleasedManifestRead = () => + { + if (commits++ > 0) + return; + System.Threading.Tasks.Task.Run(() => + { + using var writer = Open(path, Full, directory: true); + writer.Commit(batch => batch.Upsert(Deck, DeckOf("e"), E)); + }).GetAwaiter().GetResult(); + }; + + cache.PullAllBackingStoresAsync().GetAwaiter().GetResult(); + + Assert.That(cache.Get(E), Is.Not.Null, "the load retried and read the writer's generation"); + Assert.That(store.ForeignRecords.Select(record => record.Key), Is.EqualTo(new[] { "w" })); + } + + // The file decides what is carried, not what this cache loaded: a foreign record another writer added since is kept too. + [TestCase(false)] + [TestCase(true)] + public void AForeignRecordAddedAfterTheLoadIsKept(bool directory) + { + var path = Seeded("added.cc", directory); + using var cache = Open(path, DeckOnly, directory); + var z = new CultRecordKey("z"); + using (var other = Open(path, Full, directory)) + other.Commit(batch => batch.Upsert(Widget, WidgetOf("z"), z)); + var before = Stored(path, z); + + Land(cache, Write.Flush, E, DeckOf("e")); + + Assert.That(Stored(path, z), Is.EqualTo(before)); + } + + // A writer cannot see whether a foreign record holds element ids, so a store marked as holding them stays marked while it + // carries one: a write that copies any record forward keeps the mark. + [Test] + public void AMarkedHeaderStaysMarkedWhileTheStoreCarriesAForeignRecord([Values(Write.Flush, Write.Commit)] Write write) + { + var path = Seeded("marked.cc", directory: false); + var snapshot = Read(path); + snapshot.FormatVersion = CultPersistedStoreSnapshot.FormatV3; + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + + using (var cache = Open(path, DeckOnly, directory: false)) + Land(cache, write, E, DeckOf("e")); + + Assert.That(Read(path).FormatVersion, Is.EqualTo(CultPersistedStoreSnapshot.FormatV3)); + } + + private const string OtherRuntimeId = "other.runtime.deck"; + + // A deck another runtime wrote under its own id for the deck's schema, with a slot this build lacks. This build reads it + // through the catalog's schema name, but only by dropping that slot, so it does not own the id. + private string OtherRuntimeStore(string name, bool withVariant = false) + { + var path = PathOf(name); + using (var cache = Open(path, DeckOnly, directory: false)) + { + cache.Commit(batch => batch.Upsert(Deck, DeckOf("d"), D)); + if (withVariant) + cache.Commit(batch => batch.UpsertVariant(V, D, new[] { cache.Override(Deck, "Name", "v") })); + } + + var snapshot = Read(path); + var entry = snapshot.SchemaCatalog.Single(); + entry.SchemaId = OtherRuntimeId; + entry.ContentHash = OtherRuntimeId; + entry.CompatibleSchemaIds = new[] { OtherRuntimeId }; + entry.Members = entry.Members.Append(new CultSchemaMemberCatalogEntry { Slot = 99, MemberName = "Dropped", TypeName = "System.String" }).ToArray(); + foreach (var record in snapshot.Records) + record.SchemaId = OtherRuntimeId; + // An older entry that also lists the id, first in the catalog: the entry that owns the id is the one that publishes it. + snapshot.SchemaCatalog = new[] + { + new CultSchemaCatalogEntry + { + SchemaId = "decoy.older", SchemaName = "tests.decoy", SchemaVersion = "tests.decoy.v1", ContentHash = "decoy.older", + CompatibleSchemaIds = new[] { "decoy.older", OtherRuntimeId } + }, + entry + }; + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + return path; + } + + // A store whose file is gone holds nothing, foreign records included. + [Test] + public void AStoreWhoseFileIsGoneListsNoForeignRecords() + { + var path = Seeded("gone.cc", directory: false); + using var cache = Open(path, DeckOnly, directory: false); + File.Delete(path); + + cache.PullAllBackingStoresAsync().GetAwaiter().GetResult(); + + Assert.That(StoreOf(cache).ForeignRecords, Is.Empty); + } + + // The registry's public resolution still refuses an id no registered type claims, naming the schema and the id: only a store + // carries such a record. + [Test] + public void ResolvingAnUnclaimedIdThroughTheRegistryRefusesByName() + { + var entry = Full.GetRequired(Widget).ToCatalogEntry(); + + var refusal = Assert.Throws(() => DeckOnly.ResolvePersistedSchema(WidgetId, new[] { entry }))!; + + Assert.That(refusal.Message, Does.Contain("tests.foreign_widget").And.Contain(WidgetId)); + } + + private static byte[] CatalogEntryBytes(string path, string schemaId) => + CultDocumentMessagePackSerialization.SerializeSnapshot(new CultPersistedStoreSnapshot + { + SchemaCatalog = Read(path).SchemaCatalog.Where(entry => entry.SchemaId == schemaId).ToArray() + }); + + // A write of another record copies a record under an id this build does not own as the store holds it: its id, storedAt, + // bytes and the catalog entry that publishes its id. + [Test] + public void AWriteOfAnotherRecordCopiesARecordUnderAnIdThisBuildDoesNotOwn([Values] Write write) + { + var path = OtherRuntimeStore("other-runtime.cc"); + var before = Stored(path, D); + var entry = CatalogEntryBytes(path, OtherRuntimeId); + + using (var cache = Open(path, DeckOnly, directory: false)) + { + Assert.That(EmittedDocumentTypes.Read(cache.Get(D)!, "Name"), Is.EqualTo("d"), "this build reads it"); + Land(cache, write, E, DeckOf("e")); + } + + Assert.That(Stored(path, D), Is.EqualTo(before)); + Assert.That(CatalogEntryBytes(path, OtherRuntimeId), Is.EqualTo(entry)); + using var reopened = Open(path, DeckOnly, directory: false); + Assert.That(EmittedDocumentTypes.Read(reopened.Get(E)!, "Name"), Is.EqualTo("e"), "the write beside the read-only record landed"); + } + + // The entry that publishes a copied record is the one the store holds when the write lands, not the one the record loaded with. + [Test] + public void ACopiedRecordIsPublishedByTheCatalogEntryTheStoreHoldsNow([Values(Write.Flush, Write.Commit)] Write write) + { + var path = OtherRuntimeStore("redescribed.cc"); + using var cache = Open(path, DeckOnly, directory: false); + var snapshot = Read(path); + snapshot.SchemaCatalog.Single(entry => entry.SchemaId == OtherRuntimeId).SchemaVersion = "tests.foreign_deck.redescribed"; + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + var current = CatalogEntryBytes(path, OtherRuntimeId); + + Land(cache, write, E, DeckOf("e")); + + Assert.That(CatalogEntryBytes(path, OtherRuntimeId), Is.EqualTo(current)); + } + + // A write over a foreign record is refused at the write itself, in both store kinds, not first at the flush: nothing is staged + // and the cache never holds the record. + [Test] + public void AWriteOverAForeignRecordIsRefusedBeforeStaging([Values] bool directory) + { + var path = Seeded(directory ? "staged-dir.cc" : "staged-file.cc", directory); + var before = Fingerprint(path); + + using (var cache = Open(path, DeckOnly, directory)) + { + var refusal = Assert.Throws(() => cache.UpsertAsync(Deck, DeckOf(Canary), W).GetAwaiter().GetResult())!; + Assert.That((refusal.RecordKey, refusal.SchemaId), Is.EqualTo((W.Value, WidgetId))); + Assert.That(refusal.Message, Does.Not.Contain(Canary)); + Assert.That(cache.IsDirty, Is.False, "nothing was staged"); + Assert.That(cache.Get(W), Is.Null); + cache.FlushAsync().GetAwaiter().GetResult(); + } + + Assert.That(Fingerprint(path), Is.EqualTo(before)); + } + + // A record read through the catalog's schema name under an id this build neither owns nor declares is read-only: a write of + // it is refused typed before anything is staged, the store is left byte for byte, and the record is still served. + [Test] + public void ARecordReadByNameIsReadOnly([Values(Write.Flush, Write.Commit)] Write write) + { + var path = OtherRuntimeStore("read-only.cc"); + var before = Fingerprint(path); + + using (var cache = Open(path, DeckOnly, directory: false)) + { + var refusal = Assert.Throws(() => Land(cache, write, D, DeckOf("changed")))!; + Assert.That((refusal.RecordKey, refusal.SchemaId), Is.EqualTo((D.Value, OtherRuntimeId))); + Assert.That(refusal.SchemaNames, Is.EqualTo(new[] { "tests.foreign_deck" }), "it names the type that reads it"); + Assert.That(cache.IsDirty, Is.False); + Assert.That(EmittedDocumentTypes.Read(cache.Get(D)!, "Name"), Is.EqualTo("d")); + cache.FlushAsync().GetAwaiter().GetResult(); + } + + Assert.That(Fingerprint(path), Is.EqualTo(before)); + } + + public enum ReadOnlyWrite + { + Upsert, + UpsertVariantAtItsKey, + Flatten, + Remove, + CommitUpsert, + CommitRemove + } + + // Every cache write path refuses a staged write or removal of a read-only record before staging: the typed conflict, nothing + // dirty, the store byte for byte after a later flush, and the record still served. + [Test] + public void EveryWritePathRefusesAReadOnlyRecord([Values] ReadOnlyWrite path) + { + var file = OtherRuntimeStore("every-path-" + path + ".cc", withVariant: true); + var before = Fingerprint(file); + + using (var cache = Open(file, DeckOnly, directory: false)) + { + var key = path is ReadOnlyWrite.UpsertVariantAtItsKey or ReadOnlyWrite.Flatten ? V : D; + var refusal = Assert.Throws(() => + { + switch (path) + { + case ReadOnlyWrite.Upsert: + cache.UpsertAsync(Deck, DeckOf("changed"), D).GetAwaiter().GetResult(); + break; + case ReadOnlyWrite.UpsertVariantAtItsKey: + cache.UpsertVariantAsync(V, D, new[] { cache.Override(Deck, "Name", "changed") }).GetAwaiter().GetResult(); + break; + case ReadOnlyWrite.Flatten: + cache.FlattenAsync(V).GetAwaiter().GetResult(); + break; + case ReadOnlyWrite.Remove: + cache.Remove(D); + break; + case ReadOnlyWrite.CommitUpsert: + cache.Commit(batch => batch.Upsert(Deck, DeckOf("changed"), D)); + break; + case ReadOnlyWrite.CommitRemove: + cache.Commit(batch => batch.Remove(D)); + break; + } + })!; + + Assert.That((refusal.RecordKey, refusal.SchemaId), Is.EqualTo((key.Value, OtherRuntimeId))); + Assert.That(cache.IsDirty, Is.False, "nothing was staged"); + Assert.That(EmittedDocumentTypes.Read(cache.Get(D)!, "Name"), Is.EqualTo("d")); + Assert.That(EmittedDocumentTypes.Read(cache.Get(V)!, "Name"), Is.EqualTo("v")); + cache.FlushAsync().GetAwaiter().GetResult(); + } + + Assert.That(Fingerprint(file), Is.EqualTo(before)); + } + + // A type that declares the id compatible owns the record: its write lands under the type's own id and reopens. + [Test] + public void DeclaringTheIdClaimsTheRecord() + { + var path = OtherRuntimeStore("declared.cc"); + var claiming = CultDocumentRegistry.ForTypes(new[] { ClaimingDeck }); + + using (var cache = Open(path, claiming, directory: false)) + cache.Commit(batch => batch.Upsert(ClaimingDeck, New(ClaimingDeck, ("Name", "claimed")), D)); + + Assert.That(Read(path).Records.Single().SchemaId, Is.EqualTo(claiming.GetRequired(ClaimingDeck).SchemaId)); + using var reopened = Open(path, claiming, directory: false); + Assert.That(EmittedDocumentTypes.Read(reopened.Get(D)!, "Name"), Is.EqualTo("claimed")); + } + + // Whether a record is read-only is read live from the registry at the write, not fixed at load: a type registered after the + // load that declares the id makes the record writable, and the write stores it under the writing type's id, not the declarer's. + [Test] + public void ATypeRegisteredAfterLoadThatDeclaresTheIdMakesItWritable() + { + var path = OtherRuntimeStore("late.cc"); + var registry = CultDocumentRegistry.ForTypes(new[] { Deck }); + + using (var cache = Open(path, registry, directory: false)) + { + Assert.Throws(() => cache.Commit(batch => batch.Upsert(Deck, DeckOf("refused"), D))); + registry.GetRequired(LateClaimingDeck); + cache.Commit(batch => batch.Upsert(Deck, DeckOf("claimed"), D)); + } + + Assert.That(Read(path).Records.Single().SchemaId, Is.EqualTo(registry.GetRequired(Deck).SchemaId)); + using var reopened = Open(path, DeckOnly, directory: false); + Assert.That(EmittedDocumentTypes.Read(reopened.Get(D)!, "Name"), Is.EqualTo("claimed")); + } + + // A commit onto the file that involves a variant is judged on the set the file will hold. A foreign record is not decoded + // for that judgement, so it neither fails the commit nor joins the set. + [Test] + public void ACommitInvolvingAVariantIsNotRefusedByAForeignRecord() + { + var path = Seeded("variant.cc", directory: false); + var before = Stored(path, W); + using var cache = Open(path, DeckOnly, directory: false); + var v = new CultRecordKey("v"); + cache.UpsertVariantAsync(v, D, new[] { cache.Override(Deck, "Name", "v") }).GetAwaiter().GetResult(); + cache.FlushAsync().GetAwaiter().GetResult(); + + Land(cache, Write.ConditionalCommit, E, DeckOf("e")); + + Assert.That(Stored(path, W), Is.EqualTo(before)); + Assert.That(Read(path).Records.Select(record => record.Key), Is.EquivalentTo(new[] { "d", "e", "v", "w" })); + } + + // A variant's base that another writer replaced with a foreign record is gone from the set the file will hold, so a commit + // onto that file is refused as it would be if the base had been removed. + [Test] + public void ACommitOntoAFileWhereAForeignRecordReplacedAVariantsBaseIsRefused() + { + var path = Seeded("base.cc", directory: false); + using var cache = Open(path, DeckOnly, directory: false); + var v = new CultRecordKey("v"); + cache.UpsertVariantAsync(v, D, new[] { cache.Override(Deck, "Name", "v") }).GetAwaiter().GetResult(); + cache.FlushAsync().GetAwaiter().GetResult(); + var snapshot = Read(path); + var widget = snapshot.Records.Single(record => record.Key == W.Value); + snapshot.Records = snapshot.Records + .Select(record => record.Key == D.Value + ? new CultPersistedRecord { Key = D.Value, SchemaId = widget.SchemaId, StoredAt = widget.StoredAt, Payload = widget.Payload } + : record) + .ToArray(); + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + var before = Fingerprint(path); + + var refusal = Assert.Throws(Is.InstanceOf(), () => cache.Commit(batch => + { + batch.Expect(E, null); + batch.Upsert(Deck, DeckOf("e"), E); + }))!; + + Assert.That(refusal.Message, Does.Contain(v.Value).And.Contain(D.Value)); + Assert.That(Fingerprint(path), Is.EqualTo(before)); + } + + // The other writer rewrites a record under its own id, which this build reads but does not declare. + private static void Move(string path, CultRecordKey key, string name, bool directory) + { + using var other = Open(path, Moving, directory); + Assert.That(other.Commit(batch => batch.Upsert(MovingDeck, New(MovingDeck, ("Name", name)), key)), Is.True); + } + + private static void RefusedOnDisk(CultSchemaConflictException refusal, CultRecordKey key) + { + Assert.Multiple(() => + { + Assert.That((refusal.RecordKey, refusal.SchemaId), Is.EqualTo((key.Value, MovingId))); + Assert.That(refusal.SchemaNames, Is.EqualTo(new[] { "tests.foreign_deck" })); + Assert.That(refusal.Message, Does.Not.Contain(Canary), "a refusal never echoes the document it refused"); + }); + } + + // The staging half cannot see a record another writer moved to an undeclared id after this cache loaded it: the write is + // refused where it lands, under the store's lock, in both store kinds. Nothing is written, and the cache reloads the record as + // the store holds it, read-only, so the next write of it is refused before staging. + [Test] + public void AWriteOverARecordAnotherWriterMovedToAnUndeclaredIdIsRefusedAndReloadedReadOnly( + [Values] bool directory, + [Values(Write.Flush, Write.Commit)] Write write) + { + var path = PathOf("moved.cc"); + using var cache = Open(path, DeckOnly, directory); + cache.Commit(batch => batch.Upsert(Deck, DeckOf("d"), D)); + if (write == Write.Flush) + cache.UpsertAsync(Deck, DeckOf(Canary), D).GetAwaiter().GetResult(); + Move(path, D, "moved", directory); + var before = Fingerprint(path); + + var refusal = Assert.Throws(() => + { + if (write == Write.Flush) + cache.FlushAsync().GetAwaiter().GetResult(); + else + cache.Commit(batch => batch.Upsert(Deck, DeckOf(Canary), D)); + })!; + + RefusedOnDisk(refusal, D); + Assert.That(Fingerprint(path), Is.EqualTo(before)); + Assert.That(EmittedDocumentTypes.Read(cache.Get(D)!, "Name"), Is.EqualTo("moved"), "the cache serves what the store holds"); + Assert.That(cache.IsDirty, Is.False); + RefusedOnDisk(Assert.Throws(() => cache.UpsertAsync(Deck, DeckOf(Canary), D).GetAwaiter().GetResult())!, D); + Assert.That(cache.IsDirty, Is.False, "the second write was refused at staging"); + cache.FlushAsync().GetAwaiter().GetResult(); + Assert.That(Fingerprint(path), Is.EqualTo(before)); + } + + [Test] + public void AStagedRemovalOfARecordMovedToAnUndeclaredIdIsRefused([Values] bool directory) + { + var path = PathOf("moved-removal.cc"); + using var cache = Open(path, DeckOnly, directory); + cache.Commit(batch => batch.Upsert(Deck, DeckOf("d"), D)); + Assert.That(cache.Remove(D), Is.True); + Move(path, D, "moved", directory); + var moved = Stored(path, D); + + RefusedOnDisk(Assert.Throws(() => cache.FlushAsync().GetAwaiter().GetResult())!, D); + + Assert.That(Stored(path, D), Is.EqualTo(moved)); + Assert.That(EmittedDocumentTypes.Read(cache.Get(D)!, "Name"), Is.EqualTo("moved")); + Assert.That(cache.IsDirty, Is.False); + } + + // A key this cache never held, which another writer created under an undeclared id after the load: the write is refused, and + // the cache then holds the record read-only. + [Test] + public void ARecordThatAppearedUnderAnUndeclaredIdAtAWrittenKeyIsRefusedAndHeld([Values] bool directory) + { + var path = PathOf("appeared.cc"); + using var cache = Open(path, DeckOnly, directory); + cache.Commit(batch => batch.Upsert(Deck, DeckOf("e"), E)); + Move(path, D, "appeared", directory); + var before = Fingerprint(path); + + cache.UpsertAsync(Deck, DeckOf(Canary), D).GetAwaiter().GetResult(); + RefusedOnDisk(Assert.Throws(() => cache.FlushAsync().GetAwaiter().GetResult())!, D); + + Assert.That(Fingerprint(path), Is.EqualTo(before)); + Assert.That(EmittedDocumentTypes.Read(cache.Get(D)!, "Name"), Is.EqualTo("appeared")); + RefusedOnDisk(Assert.Throws(() => cache.Commit(batch => batch.Upsert(Deck, DeckOf(Canary), D)))!, D); + Assert.That(cache.IsDirty, Is.False); + } + + // A refusal reads the record it refused: in a store holding a variant, where a write lands only onto the store as this cache + // last read it, the next write of another record lands at once instead of being refused for the record already refused. + [Test] + public void AfterAnUndeclaredRefusalTheNextWriteIntoAVariantStoreLands([Values(Write.Flush, Write.Commit)] Write write) + { + var path = PathOf("variant-store.cc"); + using var cache = Open(path, DeckOnly, directory: false); + cache.Commit(batch => + { + batch.Upsert(Deck, DeckOf("d"), D); + batch.Upsert(Deck, DeckOf("e"), E); + batch.UpsertVariant(V, E, new[] { cache.Override(Deck, "Name", "v") }); + }); + Move(path, D, "moved", directory: false); + + RefusedOnDisk(Assert.Throws(() => Land(cache, write, D, DeckOf(Canary)))!, D); + Land(cache, write, new CultRecordKey("f"), DeckOf("f")); + + Assert.That(Read(path).Records.Select(record => record.Key), Is.EquivalentTo(new[] { "d", "e", "f", "v" })); + Assert.That(Read(path).Records.Single(record => record.Key == D.Value).SchemaId, Is.EqualTo(MovingId)); + } + + public enum DirectWrite + { + Push, + Delete, + CommitUpsert, + CommitDelete + } + + // The store's own write API reaches no staging refusal: a read-only record written through it directly is refused where the + // write lands, in both store kinds, and the store keeps the other writer's record under its id. + [Test] + public void ADirectStoreWriteOfAReadOnlyRecordIsRefusedAndNeverRelabelsIt([Values] bool directory, [Values] DirectWrite write) + { + var path = PathOf("direct.cc"); + Move(path, D, "theirs", directory); + var before = Fingerprint(path); + using var cache = Open(path, DeckOnly, directory); + var store = StoreOf(cache); + var mine = new CultStoredDocument(D, DateTimeOffset.UtcNow.ToString("O"), DeckOnly.GetRequired(Deck), DeckOf(Canary)); + var held = cache.GetStored(D)!; + + var refusal = Assert.Throws(() => + { + switch (write) + { + case DirectWrite.Push: + store.Push(mine); + store.PushAll(); + break; + case DirectWrite.Delete: + store.Delete(held); + store.PushAll(); + break; + case DirectWrite.CommitUpsert: + store.CommitBatch(new CultCommitRequest(new[] { mine }, Array.Empty(), Array.Empty(), Array.Empty<(CultRecordKey, string?)>(), false), wait: true); + break; + case DirectWrite.CommitDelete: + store.CommitBatch(new CultCommitRequest(Array.Empty(), new[] { held }, Array.Empty(), Array.Empty<(CultRecordKey, string?)>(), false), wait: true); + break; + } + })!; + + RefusedOnDisk(refusal, D); + Assert.That(Fingerprint(path), Is.EqualTo(before)); + Assert.That(Read(path).Records.Single().SchemaId, Is.EqualTo(MovingId), "never relabelled"); + Assert.That(store.IsDirty, Is.False); + Assert.That(EmittedDocumentTypes.Read(cache.Get(D)!, "Name"), Is.EqualTo("theirs")); + } + + // A removal of a foreign record, which the cache does not hold, is refused typed where the store judges it under its lock: at + // the commit that carries it, or the flush that follows a Remove, which itself costs nothing and returns false. The caller is + // never told a write removed what the store still carries. The store is read at that judgement, so a foreign record another + // writer added after the load is refused too, and listed foreign from then on. + [Test] + public void RemovingAForeignRecordIsRefusedNotReportedAsDone([Values] bool directory, [Values] bool batch, [Values] bool appearedAfterLoad) + { + var path = PathOf(directory ? "remove-foreign-dir.cc" : "remove-foreign-file.cc"); + using (var seed = Open(path, Full, directory)) + { + seed.Commit(stage => stage.Upsert(Deck, DeckOf("d"), D)); + if (!appearedAfterLoad) + seed.Commit(stage => stage.Upsert(Widget, WidgetOf("w"), W)); + } + + using (var cache = Open(path, DeckOnly, directory)) + { + if (appearedAfterLoad) + { + using var other = Open(path, Full, directory); + other.Commit(stage => stage.Upsert(Widget, WidgetOf("w"), W)); + } + + var before = Fingerprint(path); + var refusal = Assert.Throws(() => + { + if (batch) + cache.Commit(stage => stage.Remove(W)); + else + { + Assert.That(cache.Remove(W), Is.False, "a key the cache does not hold costs nothing at the call"); + cache.FlushAsync().GetAwaiter().GetResult(); + } + })!; + Assert.That((refusal.RecordKey, refusal.SchemaId), Is.EqualTo((W.Value, WidgetId))); + Assert.That(cache.IsDirty, Is.False); + Assert.That(StoreOf(cache).ForeignRecords.Select(record => record.Key), Is.EqualTo(new[] { "w" })); + Assert.DoesNotThrow(() => cache.FlushAsync().GetAwaiter().GetResult(), "a refusal forgets the check it refused"); + Assert.That(cache.Remove(new CultRecordKey("absent")), Is.False, "a key nothing holds is still no removal"); + Assert.That(Fingerprint(path), Is.EqualTo(before)); + } + } + + // A write that removes a key the cache does not hold is judged under the store's lock, after the load: a foreign record another + // writer committed in between refuses the whole commit, typed, and nothing the commit would have written lands. + [Test] + public void AForeignRecordThatLandsBetweenTheLoadAndTheCommitRefusesTheCommitThatRemovesIt([Values] bool directory) + { + var path = PathOf("window.cc"); + using (var seed = Open(path, Full, directory)) + seed.Commit(batch => batch.Upsert(Deck, DeckOf("d"), D)); + using var a = Open(path, DeckOnly, directory); + using (var b = Open(path, Full, directory)) + b.Commit(batch => batch.Upsert(Widget, WidgetOf("w"), W)); + var before = Fingerprint(path); + + var refusal = Assert.Throws(() => a.Commit(batch => + { + batch.Upsert(Deck, DeckOf(Canary), D); + batch.Remove(W); + }))!; + + Assert.Multiple(() => + { + Assert.That((refusal.RecordKey, refusal.SchemaId), Is.EqualTo((W.Value, WidgetId))); + Assert.That(refusal.Message, Does.Not.Contain(Canary)); + Assert.That(Fingerprint(path), Is.EqualTo(before), "nothing was written, d included"); + Assert.That(a.IsDirty, Is.False); + }); + } + + // Removing a key nothing holds reads no store: the judgement is the flush's, so a store that cannot be read costs the removal + // nothing and the flush is where it is read. + [Test] + public void RemovingAnUnheldKeyReadsNothingUntilTheFlush([Values] bool directory) + { + var path = Seeded("unreadable.cc", directory); + using var cache = Open(path, DeckOnly, directory); + var original = File.ReadAllBytes(path); + File.WriteAllBytes(path, new byte[] { 1, 2, 3, 4, 5, 6, 7, 8 }); + try + { + Assert.DoesNotThrow(() => Assert.That(cache.Remove(new CultRecordKey("absent")), Is.False)); + Assert.Throws(() => cache.FlushAsync().GetAwaiter().GetResult()); + } + finally + { + File.WriteAllBytes(path, original); + } + } + + // A checked key is judged and never removed: another writer's declared record at it stays where it is. + [Test] + public void ACheckedKeyIsNeverRemoved([Values] bool directory) + { + var path = PathOf("checked.cc"); + using (var seed = Open(path, Full, directory)) + seed.Commit(batch => batch.Upsert(Deck, DeckOf("d"), D)); + using var a = Open(path, DeckOnly, directory); + a.UpsertAsync(Deck, DeckOf("e"), E).GetAwaiter().GetResult(); + Assert.That(a.Remove(W), Is.False); + using (var b = Open(path, Full, directory)) + b.Commit(batch => batch.Upsert(Deck, DeckOf("theirs"), W)); + var theirs = Stored(path, W); + + Assert.DoesNotThrow(() => a.FlushAsync().GetAwaiter().GetResult()); + + Assert.That(Read(path).Records.Select(record => record.Key), Is.EquivalentTo(new[] { "d", "e", "w" }), "the staged write landed beside the checked key"); + Assert.That(Stored(path, W), Is.EqualTo(theirs), "the record at the checked key is still the other writer's bytes"); + using (var c = Open(path, Full, directory)) + c.Commit(batch => batch.Upsert(Widget, WidgetOf("now a widget"), W)); + Assert.DoesNotThrow(() => a.FlushAsync().GetAwaiter().GetResult(), "a flush clears what it judged: the next one has nothing to judge"); + } + + // ExpectUnchanged compares the foreign records a store carries at the versions it read them: it holds right after the load, + // and fails, writing nothing, once another writer changed one. + [Test] + public void ExpectUnchangedHoldsOverAStoreCarryingAForeignRecord([Values] bool directory) + { + var path = Seeded("expect-unchanged.cc", directory); + using var a = Open(path, DeckOnly, directory); + Assert.That(StoreOf(a).ForeignRecords.Select(record => record.Key), Is.EqualTo(new[] { "w" })); + + Assert.That(a.Commit(batch => + { + batch.ExpectUnchanged(); + batch.Upsert(Deck, DeckOf("e"), E); + }), Is.True, "nothing changed since the load"); + + using (var b = Open(path, Full, directory)) + b.Commit(batch => batch.Upsert(Widget, WidgetOf("changed"), W)); + var before = Fingerprint(path); + + Assert.That(a.Commit(batch => + { + batch.ExpectUnchanged(); + batch.Upsert(Deck, DeckOf("f"), new CultRecordKey("f")); + }), Is.False, "another writer changed the foreign record"); + Assert.That(Fingerprint(path), Is.EqualTo(before)); + } + + // A foreign key listed at the load that another writer then replaced with a record this cache reads but does not declare: the + // refused removal (judged at the flush) reloads it read-only and takes it off the foreign list. + [Test] + public void AListedForeignKeyBecomesReadOnlyWhenARemovalOfItIsRefused([Values] bool directory) + { + var path = Seeded("delist.cc", directory); + using var a = Open(path, DeckOnly, directory); + Assert.That(StoreOf(a).ForeignRecords.Select(record => record.Key), Is.EqualTo(new[] { "w" })); + // The writer declares the widget's id, so it may replace the widget with a deck record under an id a reads but does not declare. + using (var other = Open(path, CultDocumentRegistry.ForTypes(new[] { MovingDeck, Widget }), directory)) + Assert.That(other.Commit(batch => batch.Upsert(MovingDeck, New(MovingDeck, ("Name", "moved")), W)), Is.True); + + Assert.That(a.Remove(W), Is.False); + var refusal = Assert.Throws(() => a.FlushAsync().GetAwaiter().GetResult())!; + RefusedOnDisk(refusal, W); + + Assert.That(StoreOf(a).ForeignRecords, Is.Empty, "no longer foreign"); + Assert.That(a.Get(W), Is.Not.Null, "held, read-only"); + Assert.Throws(() => a.Remove(W), "refused at the call now: the cache holds it"); + } + + // A single-file store holding a variant refuses a write after another writer changed it; a changed key whose record this cache + // does not read is listed as foreign by that refusal, as a load lists it. + [Test] + public void AVariantStoreRefusalListsTheRecordItRefusedAsForeign([Values(Write.Flush, Write.Commit)] Write write) + { + var path = PathOf("variant-refusal.cc"); + var gears = CultDocumentRegistry.ForTypes(new[] { typeof(VariantTests.VariantGear) }); + var withWidget = CultDocumentRegistry.ForTypes(new[] { typeof(VariantTests.VariantGear), Widget }); + using var a = Open(path, gears, directory: false); + var baseKey = new CultRecordKey("laser"); + a.Commit(batch => + { + batch.Upsert(typeof(VariantTests.VariantGear), new VariantTests.VariantGear { Name = "laser", Power = 10, Code = "l1" }, baseKey); + batch.UpsertVariant(new CultRecordKey("laser-big"), baseKey, new[] + { + a.Override(nameof(VariantTests.VariantGear.Name), "laser big"), + a.Override(nameof(VariantTests.VariantGear.Power), 25), + a.Override(nameof(VariantTests.VariantGear.Code), "laser big") + }); + }); + using (var b = Open(path, withWidget, directory: false)) + b.Commit(batch => batch.Upsert(Widget, WidgetOf("w"), W)); + Assert.That(StoreOf(a).ForeignRecords, Is.Empty); + + var conflict = Assert.Throws(() => + { + var next = new VariantTests.VariantGear { Name = "next", Power = 1, Code = "n" }; + var key = new CultRecordKey("n"); + if (write == Write.Flush) + { + a.UpsertAsync(typeof(VariantTests.VariantGear), next, key).GetAwaiter().GetResult(); + a.FlushAsync().GetAwaiter().GetResult(); + } + else + { + a.Commit(batch => batch.Upsert(typeof(VariantTests.VariantGear), next, key)); + } + })!; + + Assert.That(conflict.ChangedKeys, Does.Contain(W.Value)); + var listed = StoreOf(a).ForeignRecords.Single(); + Assert.That((listed.Key, listed.SchemaId), Is.EqualTo((W.Value, WidgetId)), "the refused record is listed as the store holds it"); + Assert.That(a.Get(W), Is.Null); + } + + private static CacheBackingStore StoreAt(string path, bool directory, bool readOnly) => directory + ? new DirectoryMessagePackBackingStore(path, readOnly: readOnly) + : new SingleFileMessagePackBackingStore(path, readOnly); + + // A batch with nothing to write and no key to check commits through no store: a staged write waits for its own flush. + [Test] + public void ABatchWithNothingToWriteOrCheckCommitsNothingToTheStore([Values] bool directory) + { + var path = Seeded("empty-batch.cc", directory); + using var cache = Open(path, DeckOnly, directory); + cache.UpsertAsync(Deck, DeckOf("e"), E).GetAwaiter().GetResult(); + + Assert.That(cache.Commit(batch => { }), Is.True); + + Assert.That(Read(path).Records.Select(record => record.Key), Is.EquivalentTo(new[] { "d", "w" }), "e is still only staged"); + cache.FlushAsync().GetAwaiter().GetResult(); + Assert.That(Read(path).Records.Select(record => record.Key), Is.EquivalentTo(new[] { "d", "e", "w" })); + } + + // A write that only checked keys has nothing to write: no file is replaced and no generation is written, by a commit or a flush. + [Test] + public void AWriteThatOnlyCheckedKeysWritesNoFile([Values] bool directory, [Values] bool viaCommit) + { + var path = Seeded("check-only.cc", directory); + using var cache = Open(path, DeckOnly, directory); + var past = new DateTime(2001, 1, 1, 0, 0, 0, DateTimeKind.Utc); + File.SetLastWriteTimeUtc(path, past); + var before = Fingerprint(path); + + if (viaCommit) + Assert.That(cache.Commit(batch => batch.Remove(new CultRecordKey("absent"))), Is.True); + else + { + Assert.That(cache.Remove(new CultRecordKey("absent")), Is.False); + cache.FlushAsync().GetAwaiter().GetResult(); + } + + Assert.That(Fingerprint(path), Is.EqualTo(before)); + Assert.That(File.GetLastWriteTimeUtc(path), Is.EqualTo(past), "the store was not replaced"); + Assert.That(cache.IsDirty, Is.False); + } + + // A flush or commit that only checked keys clears every check it judged: the next one has nothing to judge, so it does not + // refuse a record another writer put at a checked key afterwards. + [Test] + public void AWriteThatOnlyCheckedKeysClearsTheChecksItJudged([Values] bool directory, [Values] bool viaCommit) + { + var path = Seeded("check-cleared.cc", directory); + using var cache = Open(path, DeckOnly, directory); + var staged = new CultRecordKey("staged"); + Assert.That(cache.Remove(staged), Is.False); + if (viaCommit) + Assert.That(cache.Commit(batch => batch.Remove(new CultRecordKey("requested"))), Is.True); + else + cache.FlushAsync().GetAwaiter().GetResult(); + using (var other = Open(path, Full, directory)) + other.Commit(batch => batch.Upsert(Widget, WidgetOf("now a widget"), staged)); + + Assert.DoesNotThrow(() => cache.FlushAsync().GetAwaiter().GetResult()); + } + + // A disposed cache and a disposed store judge the checks a removal staged, as their flush does. + [Test] + public void DisposingAStoreJudgesTheChecksARemovalStaged([Values] bool directory, [Values] bool viaCache) + { + var path = Seeded("dispose.cc", directory); + var cache = Open(path, DeckOnly, directory); + cache.FlushAttachedStoresOnDispose = true; + StoreOf(cache).FlushOnDispose = true; + Assert.That(cache.Remove(W), Is.False); + var before = Fingerprint(path); + + var refusal = Assert.Throws(() => + { + if (viaCache) + cache.Dispose(); + else + StoreOf(cache).Dispose(); + })!; + + Assert.That((refusal.RecordKey, refusal.SchemaId), Is.EqualTo((W.Value, WidgetId))); + Assert.That(Fingerprint(path), Is.EqualTo(before)); + try { cache.Dispose(); } catch (CultSchemaConflictException) { } + } + + // A read-only store has no lock to judge under, so it answers a removal of a key the cache does not hold at the call, from + // what it last loaded. + [Test] + public void AReadOnlyStoreRefusesTheRemovalOfAForeignRecordAtTheCall([Values] bool directory) + { + var path = Seeded("read-only.cc", directory); + using var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = DeckOnly, UseDirectoryStore = directory, ReadOnly = true }); + + var refusal = Assert.Throws(() => cache.Remove(W))!; + + Assert.That((refusal.RecordKey, refusal.SchemaId), Is.EqualTo((W.Value, WidgetId))); + Assert.That(cache.Remove(new CultRecordKey("absent")), Is.False); + } + + // A batch that only checks keys commits through the one writable store; a read-only store answers the checks from its last load. + [Test] + public void AReadOnlyStoreAnswersTheChecksFromItsView([Values] bool directory) + { + var writable = PathOf("home.cc"); + using (var seed = Open(writable, Full, directory)) + seed.Commit(batch => batch.Upsert(Deck, DeckOf("d"), D)); + var readOnly = PathOf("other.cc"); + using (var seed = Open(readOnly, CultDocumentRegistry.ForTypes(new[] { Gizmo, Widget }), directory)) + seed.Commit(batch => + { + batch.Upsert(Gizmo, New(Gizmo, ("Tag", "g")), new CultRecordKey("g")); + batch.Upsert(Widget, WidgetOf("w"), W); + }); + using var cache = new CultCache(CultDocumentRegistry.ForTypes(new[] { Deck, Gizmo })); + cache.AddBackingStore(StoreAt(writable, directory, readOnly: false), Deck); + cache.AddBackingStore(StoreAt(readOnly, directory, readOnly: true), Gizmo); + var before = Fingerprint(writable); + + var refusal = Assert.Throws(() => cache.Commit(batch => batch.Remove(W)))!; + + Assert.That((refusal.RecordKey, refusal.SchemaId), Is.EqualTo((W.Value, WidgetId))); + Assert.That(Fingerprint(writable), Is.EqualTo(before)); + Assert.That(cache.Commit(batch => batch.Remove(new CultRecordKey("absent"))), Is.True, "a key no store holds is free"); + } + + // The home store judges a check under its lock, from the record it holds then: a foreign record it listed at the load that + // another writer since replaced with a declared one no longer refuses the commit, and is not removed. + [Test] + public void TheHomeStoreJudgesACheckUnderItsLockNotFromItsLoad([Values] bool directory) + { + var path = Seeded("home-judges.cc", directory); + using var cache = Open(path, DeckOnly, directory); + Assert.That(StoreOf(cache).ForeignRecords.Select(record => record.Key), Is.EqualTo(new[] { "w" })); + using (var other = Open(path, Full, directory)) + other.Commit(batch => batch.Upsert(Deck, DeckOf("theirs"), W)); + var theirs = Stored(path, W); + + Assert.That(cache.Commit(batch => batch.Remove(W)), Is.True); + + Assert.That(Stored(path, W), Is.EqualTo(theirs), "the declared record the checked key now holds is not removed"); + } + + // A cache disposed with several stores flushes when any one of them has checks staged, not only when all do (a read-only store + // never has any). + [Test] + public void DisposingACacheFlushesWhenOnlyOneStoreHasChecksStaged([Values] bool directory) + { + var writable = Seeded("dispose-home.cc", directory); + var other = PathOf("dispose-other.cc"); + using (var seed = Open(other, CultDocumentRegistry.ForTypes(new[] { Gizmo }), directory)) + seed.Commit(batch => batch.Upsert(Gizmo, New(Gizmo, ("Tag", "g")), new CultRecordKey("g"))); + var cache = new CultCache(CultDocumentRegistry.ForTypes(new[] { Deck, Gizmo })) { FlushAttachedStoresOnDispose = true }; + cache.AddBackingStore(StoreAt(writable, directory, readOnly: false), Deck); + cache.AddBackingStore(StoreAt(other, directory, readOnly: true), Gizmo); + Assert.That(cache.Remove(W), Is.False); + + var refusal = Assert.Throws(() => cache.Dispose())!; + + Assert.That((refusal.RecordKey, refusal.SchemaId), Is.EqualTo((W.Value, WidgetId))); + try { cache.Dispose(); } catch (CultSchemaConflictException) { } + } + + // Two writable stores: home.cc holds a deck and takes Deck, other.cc holds a gizmo and takes Gizmo, and, when asked, a widget at + // w that no registered type declares, so the cache lists it foreign at the load. + private (CultCache Cache, string Home, string Other) TwoWritableStores(bool directory, bool otherListsWidget) + { + var home = PathOf("home.cc"); + using (var seed = Open(home, DeckOnly, directory)) + seed.Commit(batch => batch.Upsert(Deck, DeckOf("d"), D)); + var other = PathOf("other.cc"); + using (var seed = Open(other, CultDocumentRegistry.ForTypes(new[] { Gizmo, Widget }), directory)) + { + seed.Commit(batch => batch.Upsert(Gizmo, New(Gizmo, ("Tag", "g")), new CultRecordKey("g"))); + if (otherListsWidget) + seed.Commit(batch => batch.Upsert(Widget, WidgetOf("w"), W)); + } + + var cache = new CultCache(CultDocumentRegistry.ForTypes(new[] { Deck, Gizmo })); + cache.AddBackingStore(StoreAt(home, directory, readOnly: false), Deck); + cache.AddBackingStore(StoreAt(other, directory, readOnly: false), Gizmo); + return (cache, home, other); + } + + // Every writable store judges a batch's checks under its own lock, whichever store the batch commits through: a record another + // writer put at a checked key in a store the batch does not write, after the load, refuses the whole commit and writes nothing. + [Test] + public void EveryWritableStoreRefusesACommitThatRemovesItsLateForeignRecord([Values] bool directory, [Values] bool withWrite) + { + var (cache, home, other) = TwoWritableStores(directory, otherListsWidget: false); + using var _ = cache; + using (var late = Open(other, CultDocumentRegistry.ForTypes(new[] { Gizmo, Widget }), directory)) + late.Commit(batch => batch.Upsert(Widget, WidgetOf("late"), W)); + var homeBefore = Fingerprint(home); + var otherBefore = Fingerprint(other); + + var refusal = Assert.Throws(() => cache.Commit(batch => + { + if (withWrite) + batch.Upsert(Deck, DeckOf("d2"), new CultRecordKey("d2")); + batch.Remove(W); + }))!; + + Assert.That((refusal.RecordKey, refusal.SchemaId), Is.EqualTo((W.Value, WidgetId))); + Assert.That(Fingerprint(home), Is.EqualTo(homeBefore)); + Assert.That(Fingerprint(other), Is.EqualTo(otherBefore)); + Assert.That(Read(home).Records.Select(record => record.Key), Does.Not.Contain("d2")); + Assert.That(cache.BackingStores[1].ForeignRecords.Select(record => record.Key), Is.EqualTo(new[] { "w" })); + } + + // The non-home judgement reads the store under its lock, not its listing: a foreign record that store listed at the load, + // since replaced by a declared one, no longer refuses the commit and is not removed. + [Test] + public void ANonHomeStoreJudgesACheckUnderItsLockNotFromItsLoad([Values] bool directory) + { + var (cache, home, other) = TwoWritableStores(directory, otherListsWidget: true); + using var _ = cache; + Assert.That(cache.BackingStores[1].ForeignRecords.Select(record => record.Key), Is.EqualTo(new[] { "w" })); + using (var late = Open(other, CultDocumentRegistry.ForTypes(new[] { Gizmo, Widget }), directory)) + late.Commit(batch => batch.Upsert(Gizmo, New(Gizmo, ("Tag", "theirs")), W)); + var theirs = Stored(other, W); + + Assert.That(cache.Commit(batch => + { + batch.Upsert(Deck, DeckOf("d2"), new CultRecordKey("d2")); + batch.Remove(W); + }), Is.True); + + Assert.That(Read(home).Records.Select(record => record.Key), Does.Contain("d2")); + Assert.That(Stored(other, W), Is.EqualTo(theirs), "the declared record the checked key now holds is not removed"); + } + + // A store that is not the batch's home and cannot take its lock makes the commit contended before the home writes. + [Test] + public void ANonHomeStoreHeldByAnotherWriterMakesTryCommitContended() + { + var (cache, home, other) = TwoWritableStores(directory: false, otherListsWidget: false); + using var _ = cache; + var homeBefore = Fingerprint(home); + + using (new FileStream(other + ".lock", FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None)) + { + var outcome = cache.TryCommit(batch => + { + batch.Upsert(Deck, DeckOf("d2"), new CultRecordKey("d2")); + batch.Remove(W); + }); + + Assert.That(outcome, Is.EqualTo(CultCommitOutcome.Contended)); + } + + Assert.That(Fingerprint(home), Is.EqualTo(homeBefore)); + } + + private static void CopyStoreOver(string from, string to, bool directory) + { + if (directory) + { + var source = DirectoryMessagePackBackingStore.DefaultRecordDirectoryPath(from); + var target = DirectoryMessagePackBackingStore.DefaultRecordDirectoryPath(to); + Directory.CreateDirectory(target); + foreach (var file in Directory.GetFiles(source, "*.msgpack")) + File.Copy(file, Path.Combine(target, Path.GetFileName(file)), true); + } + File.Copy(from, to, true); + } + + private static FileStream HoldStoreLock(string path, bool directory) => directory + ? new FileStream(Path.Combine(DirectoryMessagePackBackingStore.DefaultRecordDirectoryPath(path), ".commit.lock"), FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None) + : new FileStream(path + ".lock", FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None); + + // A non-home store is judged only after the commit holds that store's own lock: a foreign record another writer lands at the + // checked key while the commit waits for the lock is refused, whichever kind the store is and whether or not the batch writes. + [Test] + public void ANonHomeStoreIsReadOnlyAfterItsLockIsTaken([Values] bool directory, [Values] bool withWrite) + { + var (cache, home, other) = TwoWritableStores(directory, otherListsWidget: false); + using var _ = cache; + var copy = PathOf("copy.cc"); + CopyStoreOver(other, copy, directory); + using (var late = Open(copy, CultDocumentRegistry.ForTypes(new[] { Gizmo, Widget }), directory)) + late.Commit(batch => batch.Upsert(Widget, WidgetOf("late"), W)); + var homeBefore = Fingerprint(home); + + Exception? thrown = null; + var held = HoldStoreLock(other, directory); + var worker = new System.Threading.Thread(() => + { + try + { + cache.Commit(batch => + { + if (withWrite) + batch.Upsert(Deck, DeckOf("d2"), new CultRecordKey("d2")); + batch.Remove(W); + }); + } + catch (Exception ex) { thrown = ex; } + }); + worker.Start(); + // The lock wait is a Thread.Sleep retry loop, so the worker sleeping is the worker waiting for the held lock. + Assert.That(System.Threading.SpinWait.SpinUntil( + () => (worker.ThreadState & System.Threading.ThreadState.WaitSleepJoin) != 0, TimeSpan.FromSeconds(10)), Is.True, "the commit waits on the held lock"); + CopyStoreOver(copy, other, directory); + var otherAfterSwap = Fingerprint(other); + held.Dispose(); + + Assert.That(worker.Join(TimeSpan.FromSeconds(40)), Is.True, "the commit finished"); + Assert.That(thrown, Is.TypeOf()); + Assert.That(Fingerprint(home), Is.EqualTo(homeBefore), "the home is unwritten"); + Assert.That(Fingerprint(other), Is.EqualTo(otherAfterSwap), "the non-home store is unwritten"); + } + + // A directory store judging a check honours wait=false: its lease held by another writer makes TryCommit contended at once. + [Test] + public void ADirectoryNonHomeStoreWhoseLeaseIsHeldMakesTryCommitContended() + { + var (cache, home, other) = TwoWritableStores(directory: true, otherListsWidget: false); + using var _ = cache; + var homeBefore = Fingerprint(home); + + CultCommitOutcome outcome; + using (HoldStoreLock(other, directory: true)) + outcome = cache.TryCommit(batch => + { + batch.Upsert(Deck, DeckOf("d2"), new CultRecordKey("d2")); + batch.Remove(W); + }); + + Assert.That(outcome, Is.EqualTo(CultCommitOutcome.Contended)); + Assert.That(Fingerprint(home), Is.EqualTo(homeBefore)); + } + + // A non-home store the cache attached before anything was written to it has no directory and no lease yet: judging a check + // there creates what it locks, finds nothing, and writes no store. + [Test] + public void ANonHomeStoreNothingWasEverWrittenToJudgesAChecksOnlyCommit([Values] bool directory) + { + var home = PathOf("home.cc"); + using (var seed = Open(home, DeckOnly, directory)) + seed.Commit(batch => batch.Upsert(Deck, DeckOf("d"), D)); + var other = PathOf("fresh" + Path.DirectorySeparatorChar + "other.cc"); + using var cache = new CultCache(CultDocumentRegistry.ForTypes(new[] { Deck, Gizmo })); + cache.AddBackingStore(StoreAt(home, directory, readOnly: false), Deck); + cache.AddBackingStore(StoreAt(other, directory, readOnly: false), Gizmo); + + Assert.That(cache.Commit(batch => + { + batch.Upsert(Deck, DeckOf("d2"), new CultRecordKey("d2")); + batch.Remove(W); + }), Is.True); + + Assert.That(Read(home).Records.Select(record => record.Key), Does.Contain("d2")); + Assert.That(File.Exists(other), Is.False, "judging a check writes no store"); + } + + // A batch with no check asks no other store anything: a lock another writer holds on a store the batch does not write does + // not stand in its way. + [Test] + public void ABatchWithoutChecksIsNotHeldUpByAnotherStoreLock() + { + var (cache, home, other) = TwoWritableStores(directory: false, otherListsWidget: false); + using var _ = cache; + + using (new FileStream(other + ".lock", FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None)) + Assert.That(cache.TryCommit(batch => batch.Upsert(Deck, DeckOf("d2"), new CultRecordKey("d2"))), Is.EqualTo(CultCommitOutcome.Committed)); + + Assert.That(Read(home).Records.Select(record => record.Key), Does.Contain("d2")); + } + + // A foreign record a refusal lists carries the version the refusal read, so a conditional commit over the stores that + // list it holds when nothing has changed since. + [Test] + public void ExpectUnchangedHoldsAfterARefusalListedTheForeignRecord([Values] bool directory) + { + var path = PathOf("refusal-listed.cc"); + using (var seed = Open(path, Full, directory)) + seed.Commit(batch => batch.Upsert(Deck, DeckOf("d"), D)); + using var cache = Open(path, DeckOnly, directory); + using (var late = Open(path, Full, directory)) + late.Commit(batch => batch.Upsert(Widget, WidgetOf("late"), W)); + Assert.That(cache.Remove(W), Is.False); + Assert.Throws(() => cache.FlushAsync().GetAwaiter().GetResult()); + Assert.That(StoreOf(cache).ForeignRecords.Select(record => record.Key), Is.EqualTo(new[] { "w" })); + + Assert.That(cache.Commit(batch => + { + batch.ExpectUnchanged(); + batch.Upsert(Deck, DeckOf("d2"), new CultRecordKey("d2")); + }), Is.True); + } + } +} diff --git a/tests/GameCult.Caching.Tests/NumericLeafVectorTests.cs b/tests/GameCult.Caching.Tests/NumericLeafVectorTests.cs index de94c253..812109df 100644 --- a/tests/GameCult.Caching.Tests/NumericLeafVectorTests.cs +++ b/tests/GameCult.Caching.Tests/NumericLeafVectorTests.cs @@ -36,7 +36,7 @@ namespace GameCult.Caching.Tests [MessagePackObject] public sealed class ArrayStringSlot { [Key(0)] public string[]? V; } [MessagePackObject] public sealed class NullableInt32Slot { [Key(0)] public int? V; } [MessagePackObject] public sealed class ListInt32Slot { [Key(0)] public List? V; } - [MessagePackObject] public sealed class LeafInner { [Key(0)] public string N = string.Empty; [Key(1)] public float W; } + [MessagePackObject] public sealed class LeafInner { [Key(0)] public string N = string.Empty; [Key(1)] public float W; [Key(2)] [CultElementId] public string Id = string.Empty; } [MessagePackObject] public sealed class RecordSlot { [Key(0)] public LeafInner? V; } [MessagePackObject] public sealed class ListRecordSlot { [Key(0)] public List? V; } [Union(0, typeof(LeafCircle))] public interface ILeafShape { } @@ -308,7 +308,7 @@ namespace GameCult.Caching.Tests { Id = "leaves", Flag = true, Blob = new byte[] { 1, 2, 3 }, I = -7, L = 5_000_000_000, F = 0.1f, D = 0.1, MaybeI = 9, Tags = new List { "a", "b" }, Names = new[] { "x", "y" }, Any = "text", - Nested = new LeafInner { N = "inner", W = 0.1f }, Items = new List { new() { N = "one", W = 0.1f }, new() { N = "two", W = 1.5f } }, + Nested = new LeafInner { N = "inner", W = 0.1f, Id = "000000000011" }, Items = new List { new() { N = "one", W = 0.1f, Id = "000000000012" }, new() { N = "two", W = 1.5f, Id = "000000000013" } }, Shape = new LeafCircle { R = 0.1f }, Floats = new[] { 0.1f, 1.5f }, MaybeF = 0.1f, }; diff --git a/tests/GameCult.Caching.Tests/PreCut2StoreFormatTests.cs b/tests/GameCult.Caching.Tests/PreCut2StoreFormatTests.cs index 5e862724..df7ce6a4 100644 --- a/tests/GameCult.Caching.Tests/PreCut2StoreFormatTests.cs +++ b/tests/GameCult.Caching.Tests/PreCut2StoreFormatTests.cs @@ -108,7 +108,7 @@ namespace GameCult.Caching.Tests using var cache = new CultCache(); var error = Assert.Catch(() => cache.AddBackingStore(new SingleFileMessagePackBackingStore(file))); Assert.That(error, Is.Not.Null); - Assert.That(error!.ToString(), Does.Contain(nameof(NotSupportedException))); + Assert.That(error, Is.TypeOf()); return error.ToString(); } finally @@ -123,10 +123,21 @@ namespace GameCult.Caching.Tests Assert.That(Refusal("unknown-header.msgpack"), Does.Contain("cultcache.store.v9")); } + // Slots of the current catalog layout that do not decode are refused by name, as a malformed store is. + [TestCase("current-catalog-bad-canonical.msgpack")] + [TestCase("current-catalog-bad-members.msgpack")] + [TestCase("current-catalog-bad-compat.msgpack")] + [TestCase("current-catalog-bad-slot5-nil.msgpack")] + [TestCase("current-catalog-bad-slot5-int.msgpack")] + public void SingleFileRefusesAMalformedCurrentCatalogEntry(string vector) + { + Assert.That(Refusal(vector), Is.Not.Empty); + } + // The release-now-c2a-bumps-format ruling rests on this: a store stamped with a newer format version is refused at // open, so this runtime cannot rewrite it. The file stays byte-identical through the refusal and through later // writes to the cache, whose refused store was never attached. - [TestCase("cultcache.store.v3")] + [TestCase("cultcache.store.v4")] [TestCase("cultcache.store.v9")] public async Task SingleFileStampedWithANewerFormatIsRefusedAndNeverRewritten(string newerFormat) { @@ -140,7 +151,7 @@ namespace GameCult.Caching.Tests var before = File.ReadAllBytes(file); using var cache = new CultCache(); - var error = Assert.Throws(() => + var error = Assert.Throws(() => cache.AddBackingStore(new SingleFileMessagePackBackingStore(file)))!; Assert.That(error.Message, Does.Contain(newerFormat)); Assert.That(cache.BackingStores, Is.Empty); @@ -156,7 +167,7 @@ namespace GameCult.Caching.Tests } } - // Two processes: this one attached a v1 store, then a later build rewrote it as v3. A plain flush, a staged write + // Two processes: this one attached a v1 store, then a later build rewrote it as v4. A plain flush, a staged write // and a staged delete must each refuse to downgrade it, leaving the file as the other process wrote it. [Test] public async Task SingleFileFlushRefusesAStoreAnotherProcessStampedNewerSinceAttach() @@ -170,16 +181,16 @@ namespace GameCult.Caching.Tests await cache.PullAllBackingStoresAsync(); var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(file)); - snapshot.FormatVersion = "cultcache.store.v3"; + snapshot.FormatVersion = "cultcache.store.v4"; File.WriteAllBytes(file, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); var before = File.ReadAllBytes(file); - await cache.AddAsync(new PreCut2FixtureNote { Text = "staged after the other process wrote v3" }); - Assert.Throws(() => cache.FlushAllBackingStores()); + await cache.AddAsync(new PreCut2FixtureNote { Text = "staged after the other process wrote v4" }); + Assert.Throws(() => cache.FlushAllBackingStores()); Assert.That(File.ReadAllBytes(file), Is.EqualTo(before)); cache.Remove(new CultRecordKey("item:anvil")); - Assert.Throws(() => cache.BackingStores.Single().PushAll()); + Assert.Throws(() => cache.BackingStores.Single().PushAll()); Assert.That(File.ReadAllBytes(file), Is.EqualTo(before)); } finally @@ -188,9 +199,10 @@ namespace GameCult.Caching.Tests } } - // An unreadable stamp is not a newer format: a flush still heals a damaged store. + // A write set cannot merge with a disk it cannot read: a flush refuses a store whose stamp is unreadable and leaves it + // as it is. [Test] - public async Task SingleFileFlushStillHealsAStoreWhoseStampIsUnreadable() + public async Task SingleFileFlushRefusesAStoreWhoseStampIsUnreadable() { var root = CopyFixture("single-file-v1"); try @@ -199,13 +211,13 @@ namespace GameCult.Caching.Tests using var cache = new CultCache(); cache.AddBackingStore(new SingleFileMessagePackBackingStore(file)); await cache.PullAllBackingStoresAsync(); - File.WriteAllBytes(file, new byte[] { 0xc1, 0xc1 }); + var damaged = new byte[] { 0xc1, 0xc1 }; + File.WriteAllBytes(file, damaged); - await cache.AddAsync(new PreCut2FixtureNote { Text = "heals" }); - cache.FlushAllBackingStores(); + await cache.AddAsync(new PreCut2FixtureNote { Text = "refused" }); + Assert.Throws(() => cache.FlushAllBackingStores()); - Assert.That(CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(file)).FormatVersion, - Is.EqualTo("cultcache.store.v1")); + Assert.That(File.ReadAllBytes(file), Is.EqualTo(damaged)); } finally { @@ -237,15 +249,32 @@ namespace GameCult.Caching.Tests private static System.Collections.Generic.IEnumerable NewerStampOverUnreadableBody() { byte garbage = 0xc1; - yield return new TestCaseData(Raw((byte)0x93, "cultcache.store.v3", garbage, garbage)).SetName("v3 header over a bad body"); - yield return new TestCaseData(Raw((byte)0x93, System.Text.Encoding.UTF8.GetBytes("cultcache.store.v3"), garbage, garbage)).SetName("bin header v3 over a bad body"); - yield return new TestCaseData(Raw((byte)0x93, System.Text.Encoding.UTF8.GetBytes("cultcache.store.v9"), garbage, garbage)).SetName("bin header v9 over a bad body"); - yield return new TestCaseData(Raw((byte)0x93, "cultcache.store.v3", (byte)0x91, garbage, (byte)0x90)).SetName("v3 header over a bad catalog"); - yield return new TestCaseData(Raw((byte)0x92, "cultcache.store.v4", garbage)).SetName("v4 header over garbage"); + yield return new TestCaseData(Raw((byte)0x93, "cultcache.store.v4", garbage, garbage), "cultcache.store.v4").SetName("v4 header over a bad body"); + yield return new TestCaseData(Raw((byte)0x93, System.Text.Encoding.UTF8.GetBytes("cultcache.store.v3"), garbage, garbage), "binary").SetName("bin header v3 over a bad body"); + yield return new TestCaseData(Raw((byte)0x93, System.Text.Encoding.UTF8.GetBytes("cultcache.store.v9"), garbage, garbage), "binary").SetName("bin header v9 over a bad body"); + yield return new TestCaseData(Raw((byte)0x93, "cultcache.store.v4", (byte)0x91, garbage, (byte)0x90), "cultcache.store.v4").SetName("v4 header over a bad catalog"); + yield return new TestCaseData(Raw((byte)0x92, "cultcache.store.v4", garbage), "cultcache.store.v4").SetName("v4 header over garbage"); + yield return new TestCaseData(Raw((byte)0x93, "cultcache.store.v5", garbage, garbage), "cultcache.store.v5").SetName("v5 header over a bad body"); + yield return new TestCaseData(BinaryStampOverAValidBody(), "binary").SetName("bin header v1 over a valid body"); + } + + // The single-file-v1 fixture's own bytes with the stamp re-encoded from a string to bin8: nothing else differs, so only the + // stamp's encoding can refuse it. + private static byte[] BinaryStampOverAValidBody() + { + var bytes = File.ReadAllBytes(Path.Combine(AppContext.BaseDirectory, "Fixtures", "pre-cut2", "single-file-v1", "store.msgpack")); + var stamp = System.Text.Encoding.UTF8.GetBytes("cultcache.store.v1"); + var at = 1; + Assert.That(bytes[0], Is.EqualTo(0x93), "the fixture opens with a three-slot array"); + Assert.That(bytes[at], Is.EqualTo((byte)(0xa0 | stamp.Length)), "the stamp is a fixstr"); + var rebuilt = new System.Collections.Generic.List { bytes[0], 0xc4, (byte)stamp.Length }; + rebuilt.AddRange(stamp); + rebuilt.AddRange(bytes.Skip(at + 1 + stamp.Length)); + return rebuilt.ToArray(); } [TestCaseSource(nameof(NewerStampOverUnreadableBody))] - public async Task SingleFileFlushRefusesANewerStampWhateverItsBodyHolds(byte[] swapped) + public async Task SingleFileFlushRefusesANewerStampWhateverItsBodyHolds(byte[] swapped, string named) { var root = CopyFixture("single-file-v1"); try @@ -257,9 +286,11 @@ namespace GameCult.Caching.Tests File.WriteAllBytes(file, swapped); await cache.AddAsync(new PreCut2FixtureNote { Text = "must not overwrite" }); - Assert.Throws(() => cache.FlushAllBackingStores()); + var flush = Assert.Throws(() => cache.FlushAllBackingStores())!; + Assert.That(flush.Message, Does.Contain(named)); Assert.That(File.ReadAllBytes(file), Is.EqualTo(swapped)); - Assert.Throws(() => new SingleFileMessagePackBackingStore(file).PullAll()); + var pull = Assert.Throws(() => new SingleFileMessagePackBackingStore(file).PullAll())!; + Assert.That(pull.Message, Does.Contain(named)); } finally { @@ -277,11 +308,11 @@ namespace GameCult.Caching.Tests using var cache = new CultCache(); cache.AddBackingStore(new DirectoryMessagePackBackingStore(manifest)); await cache.PullAllBackingStoresAsync(); - var swapped = Raw((byte)0x93, "cultcache.store.v5.directory-content-addressed-pages", (byte)0xc1, (byte)0xc1); + var swapped = Raw((byte)0x93, "cultcache.store.v6.directory-content-addressed-pages", (byte)0xc1, (byte)0xc1); File.WriteAllBytes(manifest, swapped); await cache.AddAsync(new PreCut2FixtureNote { Text = "must not overwrite" }); - Assert.Throws(() => cache.FlushAllBackingStores()); + Assert.Throws(() => cache.FlushAllBackingStores()); Assert.That(File.ReadAllBytes(manifest), Is.EqualTo(swapped)); } finally @@ -290,12 +321,13 @@ namespace GameCult.Caching.Tests } } - // A file that is read but carries no readable stamp (empty, or cut off) is damage, not a newer store: a flush heals it. + // A file that is read but carries no readable stamp (empty, or cut off) is not a store: a flush refuses it, whatever its + // length, and leaves it as it is. [TestCase(0)] [TestCase(1)] [TestCase(10)] [TestCase(-1)] - public async Task SingleFileFlushHealsAnEmptyOrTruncatedFile(int keep) + public async Task SingleFileFlushRefusesAnEmptyOrTruncatedFile(int keep) { var root = CopyFixture("single-file-v1"); try @@ -305,14 +337,13 @@ namespace GameCult.Caching.Tests cache.AddBackingStore(new SingleFileMessagePackBackingStore(file)); await cache.PullAllBackingStoresAsync(); var whole = File.ReadAllBytes(file); - File.WriteAllBytes(file, whole.Take(keep < 0 ? whole.Length / 2 : keep).ToArray()); + var damaged = whole.Take(keep < 0 ? whole.Length / 2 : keep).ToArray(); + File.WriteAllBytes(file, damaged); - await cache.AddAsync(new PreCut2FixtureNote { Text = "heals" }); - cache.FlushAllBackingStores(); + await cache.AddAsync(new PreCut2FixtureNote { Text = "refused" }); + Assert.Throws(() => cache.FlushAllBackingStores()); - var healed = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(file)); - Assert.That(healed.FormatVersion, Is.EqualTo("cultcache.store.v1")); - Assert.That(healed.Records.Length, Is.GreaterThan(0)); + Assert.That(File.ReadAllBytes(file), Is.EqualTo(damaged)); } finally { @@ -320,23 +351,29 @@ namespace GameCult.Caching.Tests } } - // A store subclass that overrides nothing beyond its four abstract members is guarded by the base, not by an opt-in. + // A store subclass that overrides nothing beyond its abstract members is guarded by the base, not by an opt-in. private sealed class PlainCustomStore : SingleFileBackingStore { public PlainCustomStore(string path) : base(path) { } - protected override byte[] SerializeSnapshot(CultPersistedStoreSnapshot snapshot) => - CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot); + protected override byte[] SerializeStore(string formatVersion, System.Collections.Generic.IReadOnlyList catalogEntries, System.Collections.Generic.IReadOnlyList records) => + CultDocumentMessagePackSerialization.SerializeStore(formatVersion, catalogEntries, records); - protected override CultPersistedStoreSnapshot DeserializeSnapshot(byte[] data) + protected override byte[] SerializeRecord(CultPersistedRecord record) => + CultDocumentMessagePackSerialization.SerializePersistedRecord(record); + + protected override byte[] SerializeCatalogEntry(CultSchemaCatalogEntry entry) => + CultDocumentMessagePackSerialization.SerializeCatalogEntry(entry); + + protected override (CultPersistedStoreSnapshot Snapshot, byte[][] CatalogBytes, byte[][] RecordBytes) ReadStore(byte[] data) { - var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(data); - CultDocumentMessagePackSerialization.RequireSingleFileFormat(snapshot); - return snapshot; + var store = CultDocumentMessagePackSerialization.ReadStore(data); + CultDocumentMessagePackSerialization.RequireSingleFileFormat(store.Snapshot); + return store; } - protected override byte[] SerializePayload(object document) => - CultDocumentMessagePackSerialization.SerializeUntyped(document, document.GetType(), Registry); + protected override byte[] SerializePayload(object document, Type documentType) => + CultDocumentMessagePackSerialization.SerializeUntyped(document, documentType, Registry); protected override object DeserializePayload(Type documentType, byte[] payload) => CultDocumentMessagePackSerialization.DeserializeUntyped(documentType, payload, Registry); @@ -359,11 +396,11 @@ namespace GameCult.Caching.Tests using var cache = new CultCache(); cache.AddBackingStore(new PlainCustomStore(file)); await cache.PullAllBackingStoresAsync(); - StampHeader(file, "cultcache.store.v3"); + StampHeader(file, "cultcache.store.v4"); var before = File.ReadAllBytes(file); await cache.AddAsync(new PreCut2FixtureNote { Text = "staged" }); - Assert.Throws(() => cache.FlushAllBackingStores()); + Assert.Throws(() => cache.FlushAllBackingStores()); Assert.That(File.ReadAllBytes(file), Is.EqualTo(before)); } finally @@ -420,10 +457,10 @@ namespace GameCult.Caching.Tests public WriteCountingStore(string path) : base(path) { } - protected override byte[] SerializeSnapshot(CultPersistedStoreSnapshot snapshot) + protected override byte[] SerializeStore(string formatVersion, System.Collections.Generic.IReadOnlyList catalogEntries, System.Collections.Generic.IReadOnlyList records) { Serialisations++; - return base.SerializeSnapshot(snapshot); + return base.SerializeStore(formatVersion, catalogEntries, records); } } @@ -476,7 +513,7 @@ namespace GameCult.Caching.Tests } } - [TestCase("cultcache.store.v5.directory-content-addressed-pages")] + [TestCase("cultcache.store.v6.directory-content-addressed-pages")] [TestCase("cultcache.store.v9")] public async Task DirectoryWritesRefuseAStoreAnotherProcessStampedNewerSinceAttach(string newer) { @@ -498,12 +535,12 @@ namespace GameCult.Caching.Tests var before = files(); await cache.AddAsync(new PreCut2FixtureNote { Text = "staged after the other process wrote the newer format" }); - Assert.Throws(() => cache.FlushAllBackingStores()); + Assert.Throws(() => cache.FlushAllBackingStores()); Assert.That(files().Keys, Is.EquivalentTo(before.Keys)); foreach (var pair in before) Assert.That(File.ReadAllBytes(pair.Key), Is.EqualTo(pair.Value), pair.Key); cache.Remove(new CultRecordKey("item:anvil")); - Assert.Throws(() => cache.FlushAllBackingStores()); + Assert.Throws(() => cache.FlushAllBackingStores()); Assert.That(files().Keys, Is.EquivalentTo(before.Keys)); foreach (var pair in before) Assert.That(File.ReadAllBytes(pair.Key), Is.EqualTo(pair.Value), pair.Key); } @@ -554,7 +591,7 @@ namespace GameCult.Caching.Tests var bytes = new byte[] { 0x93, 0xc0, 0x90, 0x90 }; Assert.That(() => CultDocumentMessagePackSerialization.DeserializeSnapshot(bytes), - Throws.TypeOf().With.Message.Contains("format version")); + Throws.TypeOf().With.Message.Contains("format version")); } // Every runtime reads a store file the same way: a missing file is an empty store, an existing empty file is not @@ -600,7 +637,8 @@ namespace GameCult.Caching.Tests public void SingleFileRefusalEchoesAHeaderOnlyInTheKnownShape(string header, bool echoed) { var snapshot = new CultPersistedStoreSnapshot { FormatVersion = header }; - var error = Assert.Throws(() => CultDocumentMessagePackSerialization.RequireSingleFileFormat(snapshot))!; + var error = Assert.Throws(() => CultDocumentMessagePackSerialization.RequireSingleFileFormat(snapshot))!; + Assert.That(error.Message, Does.Contain($"this runtime reads {CultPersistedStoreSnapshot.FormatV1}, {CultPersistedStoreSnapshot.FormatV2} and {CultPersistedStoreSnapshot.FormatV3}.")); if (echoed) { Assert.That(error.Message, Does.Contain("format " + header + " is not")); @@ -612,6 +650,28 @@ namespace GameCult.Caching.Tests } } + // The stamp decides readability and the body never does: a newer stamp is named whatever follows it, without the body parsed. + [Test] + public void ReadStoreNamesANewerSingleFileStampWithoutParsingTheBody() + { + var garbage = (byte)0xc1; + var bytes = Raw((byte)0x93, "cultcache.store.v4", garbage, garbage); + + var error = Assert.Throws(() => + CultDocumentMessagePackSerialization.ReadStore(bytes, CultDocumentMessagePackSerialization.RequireReadableSingleFileStamp))!; + + Assert.That(error.Message, Does.Contain("format cultcache.store.v4 is not readable")); + } + + // A snapshot with no format version has no stamp to write, so serialising it is refused rather than writing a nil slot 0. + [Test] + public void SerializeSnapshotRefusesASnapshotWithNoFormatVersion() + { + var snapshot = new CultPersistedStoreSnapshot { FormatVersion = null! }; + + Assert.Throws(() => CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + } + [Test] public void V1StoreWrittenAtTheBaseCommitStillDecodesByteForByte() { @@ -646,8 +706,12 @@ namespace GameCult.Caching.Tests using var cache = new CultCache(); - Assert.That(() => cache.AddBackingStore(new DirectoryMessagePackBackingStore(manifest)), - Throws.TypeOf().With.Message.Contains(formatVersion)); + // The refusal names the manifest, and shows the header only when it is cultcache.store.v: the suffixed + // names a directory store once wrote are stored text like any other. + var shown = System.Text.RegularExpressions.Regex.IsMatch(formatVersion, @"^cultcache\.store\.v[0-9]+$"); + var refusal = Assert.Throws(() => cache.AddBackingStore(new DirectoryMessagePackBackingStore(manifest)))!; + Assert.That(refusal.Path, Is.EqualTo(manifest)); + Assert.That(refusal.Message.Contains(" is " + formatVersion + ";", StringComparison.Ordinal), Is.EqualTo(shown), refusal.Message); Assert.That(cache.BackingStores, Is.Empty); Assert.That(File.ReadAllBytes(manifest), Is.EqualTo(bytes)); } @@ -656,5 +720,86 @@ namespace GameCult.Caching.Tests if (Directory.Exists(root)) Directory.Delete(root, recursive: true); } } + + // A header is named whole only in the shape cultcache.store.v<1-6 digits>, with a suffix counted and never echoed, and + // described by its length otherwise. + [TestCase("cultcache.store.v3", "cultcache.store.v3")] + [TestCase("cultcache.store.v10", "cultcache.store.v10")] + [TestCase("cultcache.store.v40", "cultcache.store.v40")] + [TestCase("cultcache.store.v123456", "cultcache.store.v123456")] + [TestCase("cultcache.store.v1234567", "an unrecognised cultcache.store.* header of 24 bytes")] + [TestCase("cultcache.store.v", "an unrecognised cultcache.store.* header of 17 bytes")] + [TestCase("cultcache.store.vx", "an unrecognised cultcache.store.* header of 18 bytes")] + [TestCase("cultcache.store.v3.extra", "cultcache.store.v3 with a suffix of 6 bytes")] + public void DescribeHeaderNamesOnlyTheKnownShape(string header, string described) + { + Assert.That(CultDocumentMessagePackSerialization.DescribeHeader(header), Is.EqualTo(described)); + } + + [Test] + public void DescribeHeaderCountsASuffixInBytes() + { + const string suffix = ".directory-content-addressed-pages"; + var header = "cultcache.store.v6" + suffix; + + Assert.That(CultDocumentMessagePackSerialization.DescribeHeader(header), + Is.EqualTo($"cultcache.store.v6 with a suffix of {System.Text.Encoding.UTF8.GetByteCount(suffix)} bytes")); + } + + [Test] + public void DirectoryStoreNamesASuffixedNewerStampByItsLength() + { + var root = Path.Combine(Path.GetTempPath(), $"cultlib-suffix-{Guid.NewGuid():N}"); + var manifest = Path.Combine(root, "store.cc"); + Directory.CreateDirectory(root); + try + { + File.WriteAllBytes(manifest, CultDocumentMessagePackSerialization.SerializeSnapshot(new CultPersistedStoreSnapshot + { + FormatVersion = "cultcache.store.v6.directory-content-addressed-pages", + SchemaCatalog = Array.Empty(), + Records = Array.Empty() + })); + + var refusal = Assert.Throws(() => new DirectoryMessagePackBackingStore(manifest).PullAll())!; + + var suffixBytes = System.Text.Encoding.UTF8.GetByteCount(".directory-content-addressed-pages"); + Assert.That(refusal.Message, Does.Contain($" is cultcache.store.v6 with a suffix of {suffixBytes} bytes;")); + } + finally + { + if (Directory.Exists(root)) Directory.Delete(root, recursive: true); + } + } + + // An array with no stamp parses to the default v1, which no directory reads: the manifest is refused, and the pages that + // belong to the store beside it are neither rewritten nor deleted. + [Test] + public void DirectoryManifestThatIsAnEmptyArrayIsRefused() + { + var root = CopyFixture("directory-v4"); + try + { + var manifest = Path.Combine(root, "store.cc"); + var emptyArray = new byte[] { 0x90 }; + File.WriteAllBytes(manifest, emptyArray); + var pages = Directory.GetFiles(root, "*", SearchOption.AllDirectories) + .Where(file => file != manifest) + .ToDictionary(file => file, File.ReadAllBytes); + Assert.That(pages, Is.Not.Empty, "the fixture's pages exist"); + + var refusal = Assert.Throws(() => new DirectoryMessagePackBackingStore(manifest).PullAll())!; + + Assert.That(refusal.Path, Is.EqualTo(manifest)); + Assert.That(File.ReadAllBytes(manifest), Is.EqualTo(emptyArray)); + Assert.That(Directory.GetFiles(root, "*", SearchOption.AllDirectories).Where(file => file != manifest), Is.EquivalentTo(pages.Keys)); + foreach (var page in pages) + Assert.That(File.ReadAllBytes(page.Key), Is.EqualTo(page.Value), page.Key); + } + finally + { + if (Directory.Exists(root)) Directory.Delete(root, recursive: true); + } + } } } diff --git a/tests/GameCult.Caching.Tests/PythonWrittenStoreTests.cs b/tests/GameCult.Caching.Tests/PythonWrittenStoreTests.cs index 6b26e90b..f83ae98d 100644 --- a/tests/GameCult.Caching.Tests/PythonWrittenStoreTests.cs +++ b/tests/GameCult.Caching.Tests/PythonWrittenStoreTests.cs @@ -53,6 +53,7 @@ namespace GameCult.Caching.Tests Assert.That(k[4], Is.EqualTo("s")); Assert.That(doc.Nested, Is.Null); Assert.That(doc.Items.Select(i => i.N), Is.EqualTo(new[] { "one", "two" })); + Assert.That(doc.Items.Select(i => i.Id), Is.EqualTo(new[] { "000000000021", "000000000022" }), "a Python-written list of records keeps its element ids"); Assert.That(Bits(doc.Items[0].W), Is.EqualTo(Bits((float)(9007199254740992L + 536870913L)))); Assert.That(Bits(doc.Items[1].W), Is.EqualTo(Bits(1e-40f))); Assert.That(Bits(((NumericLeafVectorTests.LeafCircle)doc.Shape!).R), Is.EqualTo(Bits(0.1f))); @@ -66,6 +67,181 @@ namespace GameCult.Caching.Tests } } + private static string HeaderOf(string path) => + CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)).FormatVersion; + + private static async Task<(CultCache Cache, SingleFileMessagePackBackingStore Store)> PulledAsync(string path) + { + var cache = new CultCache(); + var store = new SingleFileMessagePackBackingStore(path); + cache.AddBackingStore(store); + await cache.PullAllBackingStoresAsync(); + return (cache, store); + } + + // cultcache-py writes element ids under an unmarked v1 header. A C# write that carries such a record forward marks the + // store, because a reader from before ids would otherwise rewrite the elements without them. + [Test] + public async Task ACSharpWriteThatCopiesAPythonRecordHoldingIdsMarksTheStore() + { + var scratch = Directory.CreateTempSubdirectory().FullName; + try + { + var path = Store("leaf-record.cc", scratch); + Assert.That(HeaderOf(path), Is.EqualTo(CultPersistedStoreSnapshot.FormatV1), "cultcache-py leaves the header unmarked"); + var (cache, store) = await PulledAsync(path); + await cache.AddAsync(new NumericLeafVectorTests.LeafRecord { Id = "other" }); + store.PushAll(); + Assert.That(HeaderOf(path), Is.EqualTo(CultPersistedStoreSnapshot.FormatV3)); + + var (reloaded, _) = await PulledAsync(path); + var py = reloaded.AllEntries.OfType().Single(record => record.Id == "py"); + Assert.That(py.Items.Select(item => item.Id), Is.EqualTo(new[] { "000000000021", "000000000022" })); + } + finally + { + Directory.Delete(scratch, true); + } + } + + // What a load saw is a fact about those bytes at that version. Another writer replaced the record since, so the copy this + // write makes is not the record the load saw, and the header is not judged by it. + [Test] + public async Task ACopiedRecordAnotherWriterChangedIsNotJudgedByTheOldRead() + { + var scratch = Directory.CreateTempSubdirectory().FullName; + try + { + var path = Store("leaf-record.cc", scratch); + var (reader, readerStore) = await PulledAsync(path); + + // Another writer replaces the record with one that holds no id. + var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + var replaced = snapshot.Records.Single(record => record.Key == "py"); + replaced.Payload = CultDocumentMessagePackSerialization.SerializeUntyped( + new NumericLeafVectorTests.LeafRecord { Id = "py" }, typeof(NumericLeafVectorTests.LeafRecord)); + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + Assert.That(HeaderOf(path), Is.EqualTo(CultPersistedStoreSnapshot.FormatV1), "the replacement holds no id"); + + await reader.AddAsync(new NumericLeafVectorTests.LeafRecord { Id = "other" }); + readerStore.PushAll(); + Assert.That(HeaderOf(path), Is.EqualTo(CultPersistedStoreSnapshot.FormatV1)); + } + finally + { + Directory.Delete(scratch, true); + } + } + + // A v1 store holding only this record c0, opened by a cache that has pulled it; then another runtime appends the python + // record (and its payload rewritten by `change`) to the file raw, header still v1, the way a writer that keeps the v1 it + // found does. Returns the open cache and its store. + private static async Task<(CultCache Cache, SingleFileMessagePackBackingStore Store)> PulledThenAppended( + string path, string pySource, Action? change = null) + { + using (var seed = new CultCache()) + { + var seedStore = new SingleFileMessagePackBackingStore(path); + seed.AddBackingStore(seedStore); + await seed.AddAsync(new NumericLeafVectorTests.LeafRecord { Id = "c0" }); + seedStore.PushAll(); + } + + Assert.That(HeaderOf(path), Is.EqualTo(CultPersistedStoreSnapshot.FormatV1)); + var opened = await PulledAsync(path); + var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + var pySnapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(pySource)); + var py = pySnapshot.Records.Single(record => record.Key == "py"); + change?.Invoke(py); + snapshot.Records = snapshot.Records.Concat(new[] { py }).ToArray(); + snapshot.SchemaCatalog = snapshot.SchemaCatalog.Concat(pySnapshot.SchemaCatalog).GroupBy(entry => entry.SchemaId).Select(group => group.First()).ToArray(); + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + Assert.That(HeaderOf(path), Is.EqualTo(CultPersistedStoreSnapshot.FormatV1), "the foreign writer keeps the v1 it found"); + return opened; + } + + // The cache never read this record: it was appended after the pull. A per-instance memory of what the pull saw cannot + // know about it; the write decides from the bytes the file holds under its lock. + [Test] + public async Task ARecordAForeignWriterAddsAfterThePullMarksTheStoreWhenCopied() + { + var scratch = Directory.CreateTempSubdirectory().FullName; + try + { + var path = Path.Combine(scratch, "after-pull.cc"); + var (cache, store) = await PulledThenAppended(path, NumericLeafVectorTests.VectorPath(Path.Combine("py-store", "leaf-record.cc"))); + await cache.AddAsync(new NumericLeafVectorTests.LeafRecord { Id = "c1" }); + store.PushAll(); + Assert.That(HeaderOf(path), Is.EqualTo(CultPersistedStoreSnapshot.FormatV3)); + + var (reloaded, _) = await PulledAsync(path); + var py = reloaded.AllEntries.OfType().Single(record => record.Id == "py"); + Assert.That(py.Items.Select(item => item.Id), Is.EqualTo(new[] { "000000000021", "000000000022" })); + } + finally + { + Directory.Delete(scratch, true); + } + } + + // Ids a load minted in memory are not on disk: the check judges the file's bytes, never the document the cache holds. + [Test] + public async Task ARecordWhoseIdsExistOnlyInMemoryLeavesTheStoreUnmarked() + { + var scratch = Directory.CreateTempSubdirectory().FullName; + try + { + var path = Path.Combine(scratch, "in-memory.cc"); + var source = Store("leaf-record.cc", scratch); + var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(source)); + var idless = new NumericLeafVectorTests.LeafRecord { Id = "py" }; + idless.Items.Add(new NumericLeafVectorTests.LeafInner { N = "one" }); + idless.Items.Add(new NumericLeafVectorTests.LeafInner { N = "two" }); + snapshot.Records.Single().Payload = CultDocumentMessagePackSerialization.SerializeUntyped(idless, typeof(NumericLeafVectorTests.LeafRecord)); + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + var before = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)).Records.Single().Payload; + + var (cache, store) = await PulledAsync(path); + Assert.That(cache.AllEntries.OfType().Single().Items.Select(item => item.Id), Has.All.Length.EqualTo(12), "the load minted ids in memory"); + await cache.AddAsync(new NumericLeafVectorTests.LeafRecord { Id = "other" }); + store.PushAll(); + + Assert.That(HeaderOf(path), Is.EqualTo(CultPersistedStoreSnapshot.FormatV1)); + var after = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)).Records.Single(record => record.Key == "py").Payload; + Assert.That(after, Is.EqualTo(before), "the copied record is the bytes the file held"); + } + finally + { + Directory.Delete(scratch, true); + } + } + + // The write-set rule copies records without interpreting them, so the check must never refuse a write: a record this + // cache claims but cannot decode is copied as its bytes and decides nothing. + [Test] + public async Task ACopiedRecordThatDoesNotDecodeIsCopiedAndDecidesNothing() + { + var scratch = Directory.CreateTempSubdirectory().FullName; + try + { + var path = Path.Combine(scratch, "undecodable.cc"); + var garbage = new byte[] { 0xc1, 0xc1, 0xc1 }; + var (cache, store) = await PulledThenAppended( + path, NumericLeafVectorTests.VectorPath(Path.Combine("py-store", "leaf-record.cc")), record => record.Payload = garbage); + + await cache.AddAsync(new NumericLeafVectorTests.LeafRecord { Id = "c1" }); + Assert.DoesNotThrow(() => store.PushAll()); + + Assert.That(HeaderOf(path), Is.EqualTo(CultPersistedStoreSnapshot.FormatV1)); + var py = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)).Records.Single(record => record.Key == "py"); + Assert.That(py.Payload, Is.EqualTo(garbage)); + } + finally + { + Directory.Delete(scratch, true); + } + } + [TestCase(3)] [TestCase(5)] [TestCase(7)] diff --git a/tests/GameCult.Caching.Tests/RefListDriftTests.cs b/tests/GameCult.Caching.Tests/RefListDriftTests.cs index 2cb52167..9aa180ea 100644 --- a/tests/GameCult.Caching.Tests/RefListDriftTests.cs +++ b/tests/GameCult.Caching.Tests/RefListDriftTests.cs @@ -106,6 +106,55 @@ namespace GameCult.Caching.Tests } } + // The store the 1.0.60 writer wrote (tests/vectors/ref-list-drift/v1-0-60-written.cc, made by OldWriterRefListProbe), or the one + // old-reader-refusal.sh gate csharp names in CULTLIB_GATE_OLD_STORE. It attaches the store, upserts alpha, beta and the list + // reversed, flushes, rereads the file and prints the REFLIST-SUMMARY line OldReaderRefListProbe prints at 2.0.1 after the same + // write; the gate diffs them, so the two must stay the same code. Ids are compared, not names: the write is what makes a record's + // schema id the one its type owns, and a record left under its persisted id is a type that cannot write it. + [Test] + public async Task A_1_0_60_Written_Store_Opens_Like_2_0_1() + { + var source = Environment.GetEnvironmentVariable("CULTLIB_GATE_OLD_STORE"); + var copy = Path.Combine(Path.GetTempPath(), $"reflist-1060-{Guid.NewGuid():N}.cc"); + File.Copy(string.IsNullOrEmpty(source) ? VectorPath("v1-0-60-written.cc") : source, copy); + try + { + string[] warnings; + using (var cache = new CultCache()) + { + var store = new SingleFileMessagePackBackingStore(copy); + cache.AddBackingStore(store); + await cache.PullAllBackingStoresAsync(); + warnings = store.LastSchemaMigrationReports.SelectMany(report => report.Warnings).Select(warning => warning.Code).Distinct().OrderBy(code => code, StringComparer.Ordinal).ToArray(); + Assert.That(warnings, Does.Contain("gained_reference_semantics")); + var holder = cache.GetAll().Single(); + Assert.That(holder.Items.Select(item => item.Key.Value), Is.EqualTo(new[] { "alpha", "beta" })); + + // The write both runtimes make: the targets as they are, the holder with its list reversed. + foreach (var target in cache.GetAll().ToArray()) await cache.UpsertAsync(target, cache.TryGetHandle(target)); + holder.Items = holder.Items.AsEnumerable().Reverse().ToArray(); + await cache.UpsertAsync(holder, cache.TryGetHandle(holder)); + await cache.FlushAsync(); + } + + var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(copy)); + var records = snapshot.Records.OrderBy(record => record.Key, StringComparer.Ordinal).Select(record => record.Key + "=" + record.SchemaId).ToArray(); + var listId = snapshot.Records.Single(record => record.Key == "list").SchemaId; + var slot = snapshot.SchemaCatalog.Single(entry => entry.SchemaId == listId).Members.Single(member => member.MemberName == "Items"); + using var reopened = new CultCache(); + reopened.AddBackingStore(new SingleFileMessagePackBackingStore(copy)); + await reopened.PullAllBackingStoresAsync(); + var items = reopened.GetAll().Single().Items.Select(item => item.Key.Value).ToArray(); + Assert.That(items, Is.EqualTo(new[] { "beta", "alpha" })); + Assert.That(reopened.Get(reopened.GetAll().Single().Items[0].Key)!.Name, Is.EqualTo("beta")); + TestContext.Out.WriteLine("REFLIST-SUMMARY " + System.Text.Json.JsonSerializer.Serialize(new { records, items, itemsSlot = new { slot.IsReference, slot.TargetSchemaName }, warnings, header = snapshot.FormatVersion })); + } + finally + { + File.Delete(copy); + } + } + private static CultSchemaMigrationReport Resolve(Action persistedItems) { var catalog = CultDocumentRegistry.Shared.GetRequired().ToCatalogEntry(); diff --git a/tests/GameCult.Caching.Tests/RestampOnWriteTests.cs b/tests/GameCult.Caching.Tests/RestampOnWriteTests.cs new file mode 100644 index 00000000..b90f4336 --- /dev/null +++ b/tests/GameCult.Caching.Tests/RestampOnWriteTests.cs @@ -0,0 +1,823 @@ +#nullable enable +using System; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Threading.Tasks; +using GameCult.Caching.MessagePack; +using MessagePack; +using NUnit.Framework; + +namespace GameCult.Caching.Tests +{ + // variants:ruling:drift-record-write-ownership: a writer owns a record stored under id X when it declares X, or when the type + // that reads X reads it whole; it then writes, relabels and removes it, and a write stores it under the type's own id. A record + // a type reads only by dropping persisted slots stays read-only until its id is declared compatible. + public class RestampOnWriteTests + { + // One schema name at three versions of its type and nothing declared: Grown reads Narrow's records whole (it adds a slot), + // and reads Wide's records only by dropping slot 3. + [CultDocument("tests.restamp_note", "tests.restamp_note.v1")] + [MessagePackObject(AllowPrivate = true)] + internal sealed class NoteNarrow + { + [Key(0)] [CultName] public string Name = string.Empty; + [Key(1)] public int Value; + } + + [CultDocument("tests.restamp_note", "tests.restamp_note.v2")] + [MessagePackObject(AllowPrivate = true)] + internal sealed class NoteGrown + { + [Key(0)] [CultName] public string Name = string.Empty; + [Key(1)] public int Value; + [Key(2)] public string Colour = string.Empty; + } + + [CultDocument("tests.restamp_note", "tests.restamp_note.v3")] + [MessagePackObject(AllowPrivate = true)] + internal sealed class NoteWide + { + [Key(0)] [CultName] public string Name = string.Empty; + [Key(1)] public int Value; + [Key(2)] public string Colour = string.Empty; + [Key(3)] public string Extra = string.Empty; + } + + // Another schema with Narrow's exact shape: never the same type, whatever the members say. + [CultDocument("tests.restamp_other", "tests.restamp_other.v1")] + [MessagePackObject(AllowPrivate = true)] + internal sealed class NoteOther + { + [Key(0)] [CultName] public string Name = string.Empty; + [Key(1)] public int Value; + } + + // Reads a Narrow-shaped entry only by dropping Value. + [CultDocument("tests.restamp_short", "tests.restamp_short.v1")] + [MessagePackObject(AllowPrivate = true)] + internal sealed class NoteShort + { + [Key(0)] [CultName] public string Name = string.Empty; + } + + // Another schema with Narrow's shape that lists an old id: it claims records stored under that id. + [CultDocument("tests.restamp_lister", "tests.restamp_lister.v1", CompatibleSchemaIds = new[] { ListerOldId })] + [MessagePackObject(AllowPrivate = true)] + internal sealed class NoteLister + { + [Key(0)] [CultName] public string Name = string.Empty; + [Key(1)] public int Value; + } + + private const string ListerOldId = "tests.restamp.lister_old"; + + // Grown with the lossy id declared: declaration is the consent to the loss. + [CultDocument("tests.restamp_note", "tests.restamp_note.v4", CompatibleSchemaIds = new[] { WideId })] + [MessagePackObject(AllowPrivate = true)] + internal sealed class NoteGrownDeclaring + { + [Key(0)] [CultName] public string Name = string.Empty; + [Key(1)] public int Value; + [Key(2)] public string Colour = string.Empty; + } + + private const string WideId = "sha256:82c2b721aceb56e00288baab590fabcdd82c3bda44400f7530fa9e29c4e96252"; + + private static readonly CultDocumentRegistry NarrowRegistry = CultDocumentRegistry.ForTypes(new[] { typeof(NoteNarrow) }); + private static readonly CultDocumentRegistry GrownRegistry = CultDocumentRegistry.ForTypes(new[] { typeof(NoteGrown) }); + private static readonly CultDocumentRegistry WideRegistry = CultDocumentRegistry.ForTypes(new[] { typeof(NoteWide) }); + private static readonly CultDocumentRegistry DeclaringRegistry = CultDocumentRegistry.ForTypes(new[] { typeof(NoteGrownDeclaring) }); + private static readonly CultRecordKey Note = new("note"); + private static readonly CultRecordKey Other = new("other"); + + private string _directory = string.Empty; + + [SetUp] + public void CreateDirectory() + { + _directory = Path.Combine(Path.GetTempPath(), $"cultlib-restamp-{Guid.NewGuid():N}"); + Directory.CreateDirectory(_directory); + } + + [TearDown] + public void DeleteDirectory() + { + if (Directory.Exists(_directory)) + Directory.Delete(_directory, recursive: true); + } + + public static readonly object[] StoreKinds = { false, true }; + + private CacheBackingStore Store(string path, bool directory) => + directory ? new DirectoryMessagePackBackingStore(path) : new SingleFileMessagePackBackingStore(path); + + private CultCache Open(string path, bool directory, CultDocumentRegistry registry) + { + var cache = new CultCache(registry); + cache.AddBackingStore(Store(path, directory)); + return cache; + } + + // The schema id each key is stored under, read back through a fresh cache that declares everything it sees. + private string StoredIdOf(string path, bool directory, CultDocumentRegistry registry, CultRecordKey key) + { + using var cache = Open(path, directory, registry); + return cache.GetStored(key)!.StoredSchemaId; + } + + private static string Fingerprint(string path) + { + using var sha = SHA256.Create(); + if (File.Exists(path)) + return Convert.ToHexString(sha.ComputeHash(File.ReadAllBytes(path))); + var parts = Directory.GetFiles(path, "*", SearchOption.AllDirectories) + .OrderBy(file => file, StringComparer.Ordinal) + .Select(file => Path.GetRelativePath(path, file) + ":" + Convert.ToHexString(sha.ComputeHash(File.ReadAllBytes(file)))); + return string.Join("|", parts); + } + + // Two records written by a writer that knew Narrow: the one under test and a bystander. + private string SeedNarrow(bool directory) + { + var path = Path.Combine(_directory, directory ? "store" : "store.cc"); + using var cache = Open(path, directory, NarrowRegistry); + Assert.That(cache.Commit(batch => + { + batch.Upsert(new NoteNarrow { Name = "note", Value = 1 }, new CultRecordHandle(Note)); + batch.Upsert(new NoteNarrow { Name = "other", Value = 7 }, new CultRecordHandle(Other)); + }), Is.True); + return path; + } + + private string SeedWide(bool directory) + { + var path = Path.Combine(_directory, directory ? "store" : "store.cc"); + using var cache = Open(path, directory, WideRegistry); + Assert.That(cache.Commit(batch => batch.Upsert(new NoteWide { Name = "note", Value = 1, Extra = "kept?" }, new CultRecordHandle(Note))), Is.True); + return path; + } + + [TestCaseSource(nameof(StoreKinds))] + public void ADriftRecordReadWholeIsWrittenUnderItsTypesId(bool directory) + { + var path = SeedNarrow(directory); + var narrowId = NarrowRegistry.GetRequired().SchemaId; + var grownId = GrownRegistry.GetRequired().SchemaId; + Assert.That(StoredIdOf(path, directory, NarrowRegistry, Note), Is.EqualTo(narrowId)); + + using (var cache = Open(path, directory, GrownRegistry)) + { + Assert.That(cache.Get(Note)!.Value, Is.EqualTo(1)); + Assert.That(cache.Commit(batch => batch.Upsert(new NoteGrown { Name = "note", Value = 2, Colour = "red" }, new CultRecordHandle(Note))), Is.True); + } + + Assert.That(StoredIdOf(path, directory, GrownRegistry, Note), Is.EqualTo(grownId), "the written record moved to its type's id"); + Assert.That(StoredIdOf(path, directory, NarrowRegistry, Other), Is.EqualTo(narrowId), "the bystander kept its persisted id (write set)"); + using var reopened = Open(path, directory, GrownRegistry); + Assert.That(reopened.Get(Note)!.Colour, Is.EqualTo("red")); + Assert.That(reopened.Get(Other)!.Value, Is.EqualTo(7)); + } + + [TestCaseSource(nameof(StoreKinds))] + public void RemovingADriftRecordReadWholeSucceeds(bool directory) + { + var path = SeedNarrow(directory); + using (var cache = Open(path, directory, GrownRegistry)) + Assert.That(cache.Commit(batch => batch.Remove(Note)), Is.True); + + using var reopened = Open(path, directory, GrownRegistry); + Assert.That(reopened.Get(Note), Is.Null); + Assert.That(reopened.Get(Other), Is.Not.Null); + } + + [TestCaseSource(nameof(StoreKinds))] + public void ADriftRecordReadOnlyByDroppingASlotStaysReadOnly(bool directory) + { + var path = SeedWide(directory); + var before = Fingerprint(path); + + using var cache = Open(path, directory, GrownRegistry); + Assert.That(cache.Get(Note), Is.Not.Null, "a read-only record is still served"); + var upsert = Assert.Throws(() => + cache.Commit(batch => batch.Upsert(new NoteGrown { Name = "note", Value = 2 }, new CultRecordHandle(Note))))!; + Assert.That(upsert.RecordKey, Is.EqualTo("note")); + Assert.That(upsert.SchemaId, Is.EqualTo(WideRegistry.GetRequired().SchemaId)); + Assert.That(upsert.Message, Does.Contain("would drop persisted slots").And.Contain("to accept the loss."), "the refusal names the narrowed rule"); + var remove = Assert.Throws(() => cache.Commit(batch => batch.Remove(Note)))!; + Assert.That(remove.RecordKey, Is.EqualTo("note")); + Assert.That(Fingerprint(path), Is.EqualTo(before), "a refused write leaves the store as it was"); + } + + // A held record read only by dropping a slot is refused before anything is staged: with the store's lock held by another + // writer, the write is refused typed instead of waiting on the lock and reporting contention. + [TestCaseSource(nameof(StoreKinds))] + public void ALossyDriftRecordIsRefusedBeforeTheStoresLockIsAsked(bool directory) + { + var path = SeedWide(directory); + using var cache = Open(path, directory, GrownRegistry); + using var held = directory + ? new FileStream(Path.Combine(DirectoryMessagePackBackingStore.DefaultRecordDirectoryPath(path), ".commit.lock"), FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None) + : new FileStream(path + ".lock", FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None); + + Assert.Throws(() => + cache.TryCommit(batch => batch.Upsert(new NoteGrown { Name = "note", Value = 2 }, new CultRecordHandle(Note)))); + } + + [TestCaseSource(nameof(StoreKinds))] + public void ADeclaredLossyDriftRecordIsWritable(bool directory) + { + var path = SeedWide(directory); + var wideId = WideRegistry.GetRequired().SchemaId; + Assert.That(wideId, Is.EqualTo(WideId), "the declaration names the lossy record's id: " + wideId); + + using (var cache = Open(path, directory, DeclaringRegistry)) + Assert.That(cache.Commit(batch => batch.Upsert(new NoteGrownDeclaring { Name = "note", Value = 2 }, new CultRecordHandle(Note))), Is.True); + + Assert.That(StoredIdOf(path, directory, DeclaringRegistry, Note), Is.EqualTo(DeclaringRegistry.GetRequired().SchemaId)); + } + + // Another writer rewrites the record after this cache loaded it: ownership is judged again under the store's lock. + [TestCaseSource(nameof(StoreKinds))] + public void AnotherWritersDriftRecordIsJudgedUnderTheLock(bool directory) + { + var path = SeedNarrow(directory); + using var cache = Open(path, directory, GrownRegistry); + Assert.That(cache.Get(Note), Is.Not.Null); + + // A writer that knows Wide replaces the record: this cache's loaded copy was read whole, the durable one is not. + using (var other = Open(path, directory, WideRegistry)) + { + Assert.That(other.Commit(batch => batch.Upsert(new NoteWide { Name = "note", Value = 3, Extra = "x" }, new CultRecordHandle(Note))), Is.True); + } + + var refusal = Assert.Throws(() => + cache.Commit(batch => batch.Upsert(new NoteGrown { Name = "note", Value = 4 }, new CultRecordHandle(Note))))!; + Assert.That(refusal.RecordKey, Is.EqualTo("note")); + Assert.That(StoredIdOf(path, directory, WideRegistry, Note), Is.EqualTo(WideRegistry.GetRequired().SchemaId)); + } + + [Test] + public void ACachelessWriteRestampsADriftRecordOfItsSchema() + { + var path = SeedNarrow(directory: false); + new SingleFileMessagePackBackingStore(path).WriteDocument(Note, new NoteGrown { Name = "note", Value = 2 }, registry: GrownRegistry); + + Assert.That(StoredIdOf(path, false, GrownRegistry, Note), Is.EqualTo(GrownRegistry.GetRequired().SchemaId)); + Assert.That(StoredIdOf(path, false, NarrowRegistry, Other), Is.EqualTo(NarrowRegistry.GetRequired().SchemaId)); + } + + [Test] + public void ACachelessWriteOverALossyDriftRecordIsRefusedAndTheFileIsUntouched() + { + var path = SeedWide(directory: false); + var before = Fingerprint(path); + var refusal = Assert.Throws(() => + new SingleFileMessagePackBackingStore(path).WriteDocument(Note, new NoteGrown { Name = "note", Value = 2 }, registry: GrownRegistry))!; + Assert.That(refusal.Message, Does.Contain("reads it whole").And.Contain("to accept the loss of its unread slots."), "the refusal names the narrowed rule"); + Assert.That(Fingerprint(path), Is.EqualTo(before)); + } + + // A cacheless writer's entry that lists the record's id compatible consents to the loss, as a cache's declaration does. + [Test] + public void ACachelessWriteByAnEntryListingTheIdCompatibleOverwritesALossyDriftRecord() + { + var path = SeedWide(directory: false); + new SingleFileMessagePackBackingStore(path).WriteDocument(Note, new NoteGrownDeclaring { Name = "note", Value = 2 }, registry: DeclaringRegistry); + + Assert.That(StoredIdOf(path, false, DeclaringRegistry, Note), Is.EqualTo(DeclaringRegistry.GetRequired().SchemaId)); + } + + // The type that would read the id cannot be named (two registered versions share the schema name and the id declares neither), + // so the record is not owned and a write over it is refused typed, not as the resolver's own error. + [TestCaseSource(nameof(StoreKinds))] + public void ARecordNoRegisteredTypeCanBeNamedForIsNotOwned(bool directory) + { + var path = SeedNarrow(directory); + var both = CultDocumentRegistry.ForTypes(new[] { typeof(NoteNarrow), typeof(NoteGrown) }); + using var cache = Open(path, directory, both); + using (var other = Open(path, directory, WideRegistry)) + Assert.That(other.Commit(batch => batch.Upsert(new NoteWide { Name = "note", Value = 3 }, new CultRecordHandle(Note))), Is.True); + + Assert.Throws(() => + cache.Commit(batch => batch.Upsert(new NoteGrown { Name = "note", Value = 4 }, new CultRecordHandle(Note)))); + } + + [TestCaseSource(nameof(StoreKinds))] + public void ExpectAgreesWithRestamp(bool directory) + { + var path = SeedNarrow(directory); + var third = new CultRecordKey("third"); + using var cache = Open(path, directory, GrownRegistry); + + Assert.That(cache.Commit(batch => + { + batch.Expect(Note, cache.Get(Note)); + batch.ExpectUnchanged(); + batch.Upsert(new NoteGrown { Name = "third" }, new CultRecordHandle(third)); + }), Is.True, "an unchanged drift record satisfies Expect and ExpectUnchanged"); + + using var second = Open(path, directory, GrownRegistry); + Assert.That(cache.Commit(batch => + { + batch.Expect(Note, cache.Get(Note)); + batch.Upsert(new NoteGrown { Name = "note", Value = 2 }, new CultRecordHandle(Note)); + }), Is.True, "a batch that Expects the drift record and upserts it commits"); + Assert.That(StoredIdOf(path, directory, GrownRegistry, Note), Is.EqualTo(GrownRegistry.GetRequired().SchemaId), "and restamps it"); + + Assert.That(cache.Commit(batch => + { + batch.ExpectUnchanged(); + batch.Upsert(new NoteGrown { Name = "fourth" }, new CultRecordHandle(new CultRecordKey("fourth"))); + }), Is.True, "the writer's later ExpectUnchanged holds"); + Assert.That(second.Commit(batch => + { + batch.Expect(Note, second.Get(Note)); + batch.Upsert(new NoteGrown { Name = "note", Value = 3 }, new CultRecordHandle(Note)); + }), Is.False, "a cache loaded before the restamp fails its Expect: the bytes changed"); + } + + private static readonly CultRecordKey Base = new("base"); + private static readonly CultRecordKey Delta = new("delta"); + + private CultCache OpenWithCodec(string path, bool directory, CultDocumentRegistry registry) + { + var cache = new CultCache(registry, CultCacheMessagePack.CreateCodec(registry)); + cache.AddBackingStore(Store(path, directory)); + return cache; + } + + private string StoredIdOfVariantStore(string path, CultDocumentRegistry registry, CultRecordKey key) + { + using var cache = OpenWithCodec(path, false, registry); + return cache.GetStored(key)!.StoredSchemaId; + } + + // A base and a variant of it, written by a writer that knew Narrow; Grown then reads the variant whole as compatible drift. + private string SeedNarrowVariant() + { + var path = Path.Combine(_directory, "store.cc"); + using var cache = OpenWithCodec(path, false, NarrowRegistry); + Assert.That(cache.Commit(batch => batch.Upsert(new NoteNarrow { Name = "base", Value = 1 }, new CultRecordHandle(Base))), Is.True); + Assert.That(cache.Commit(batch => batch.UpsertVariant(Delta, Base, new[] { cache.Override("Name", "delta"), cache.Override("Value", 5) })), Is.True); + return path; + } + + // Variants live in a single file (the directory store refuses them). The staging half asks the flag ToStoredDocument set on a loaded variant; Resolved and Flatten copies are never asked (only the store's loaded view carries the read-whole flag): an owned + // drift variant is rewritten, flattened and removed like a plain drift record, restamping only what is written. + [Test] + public async Task AnOwnedDriftVariantIsRewritten() + { + var path = SeedNarrowVariant(); + using (var cache = OpenWithCodec(path, false, GrownRegistry)) + { + await cache.PullAllBackingStoresAsync(); + Assert.That(cache.Commit(batch => batch.UpsertVariant(Delta, Base, new[] { cache.Override("Name", "delta"), cache.Override("Value", 6) })), Is.True); + } + + Assert.That(StoredIdOfVariantStore(path, GrownRegistry, Delta), Is.EqualTo(GrownRegistry.GetRequired().SchemaId)); + Assert.That(StoredIdOfVariantStore(path, NarrowRegistry, Base), Is.EqualTo(NarrowRegistry.GetRequired().SchemaId), "the base was not written"); + } + + [Test] + public async Task AnOwnedDriftVariantIsFlattened() + { + var path = SeedNarrowVariant(); + using (var cache = OpenWithCodec(path, false, GrownRegistry)) + { + await cache.PullAllBackingStoresAsync(); + Assert.That(cache.Commit(batch => batch.Flatten(Delta)), Is.True); + } + + Assert.That(StoredIdOfVariantStore(path, GrownRegistry, Delta), Is.EqualTo(GrownRegistry.GetRequired().SchemaId)); + } + + [Test] + public async Task AnOwnedDriftVariantIsRemoved() + { + var path = SeedNarrowVariant(); + using (var cache = OpenWithCodec(path, false, GrownRegistry)) + { + await cache.PullAllBackingStoresAsync(); + Assert.That(cache.Commit(batch => batch.Remove(Delta)), Is.True); + } + + using var reopened = OpenWithCodec(path, false, GrownRegistry); + await reopened.PullAllBackingStoresAsync(); + Assert.That(reopened.GetStored(Delta), Is.Null); + Assert.That(reopened.GetStored(Base), Is.Not.Null); + } + + // A store whose entry is a newer writer's rename of Narrow's schema, with identical members and the given compatible ids. + private string SeedRenamed(bool listsNarrow, string? alsoLists = null, string name = "tests.restamp_note_renamed") + { + var path = SeedNarrow(directory: false); + var snapshot = Snapshot(path); + var narrowId = NarrowRegistry.GetRequired().SchemaId; + var entry = snapshot.SchemaCatalog.Single(); + entry.SchemaId = entry.ContentHash = "tests.restamp.renamed"; + entry.SchemaName = name; + entry.SchemaVersion = "tests.restamp_note_renamed.v2"; + entry.CompatibleSchemaIds = new[] { "tests.restamp.renamed" } + .Concat(listsNarrow ? new[] { narrowId } : Array.Empty()) + .Concat(alsoLists == null ? Array.Empty() : new[] { alsoLists }) + .ToArray(); + foreach (var record in snapshot.Records) + record.SchemaId = entry.SchemaId; + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + return path; + } + + // One writer of NoteNarrow, with a cache or without one: the same Owns answer for the same record. + private void WriteNarrow(string path, bool cacheless, CultDocumentRegistry? registry = null) + { + registry ??= NarrowRegistry; + if (cacheless) + { + new SingleFileMessagePackBackingStore(path).WriteDocument(Note, new NoteNarrow { Name = "note", Value = 2 }, registry: registry); + return; + } + + using var cache = Open(path, false, registry); + cache.PullAllBackingStoresAsync().GetAwaiter().GetResult(); + Assert.That(cache.Commit(batch => batch.Upsert(new NoteNarrow { Name = "note", Value = 2 }, new CultRecordHandle(Note))), Is.True); + } + + [TestCase(false)] + [TestCase(true)] + public void ARenamedEntryListingTheWritersIdIsOwnedByCacheAndCachelessWriterAlike(bool cacheless) + { + var path = SeedRenamed(listsNarrow: true); + WriteNarrow(path, cacheless); + + Assert.That(Snapshot(path).Records.Single(record => record.Key == "note").SchemaId, Is.EqualTo(NarrowRegistry.GetRequired().SchemaId)); + } + + [TestCase(false)] + [TestCase(true)] + public void ARenamedEntryNotListingTheWritersIdIsRefusedByCacheAndCachelessWriterAlike(bool cacheless) + { + var path = SeedRenamed(listsNarrow: false); + var before = Fingerprint(path); + Assert.Throws(() => WriteNarrow(path, cacheless)); + Assert.That(Fingerprint(path), Is.EqualTo(before)); + } + + // A writer of one schema never overwrites and retypes a record of another schema, however alike their members. + [TestCase(false)] + [TestCase(true)] + public void AWriterOfOneSchemaNeverRetypesASameShapeRecordOfAnother(bool cacheless) + { + var path = Path.Combine(_directory, "store.cc"); + var otherRegistry = CultDocumentRegistry.ForTypes(new[] { typeof(NoteOther) }); + using (var seed = Open(path, false, otherRegistry)) + Assert.That(seed.Commit(batch => batch.Upsert(new NoteOther { Name = "note", Value = 1 }, new CultRecordHandle(Note))), Is.True); + var before = Fingerprint(path); + + Assert.Throws(() => WriteNarrow(path, cacheless)); + Assert.That(Fingerprint(path), Is.EqualTo(before)); + } + + private static readonly CultDocumentRegistry NarrowShortRegistry = CultDocumentRegistry.ForTypes(new[] { typeof(NoteNarrow), typeof(NoteShort) }); + + // Which entry reads a persisted entry is one precedence: a cacheless writer lent a registry picks the claimant that registry + // picks, never one by schema name over its own entry. Here the entry is named and shaped like Narrow but lists Short's id. + [TestCase(false)] + [TestCase(true)] + public void ACachelessWriterPicksTheClaimantItsRegistryPicks(bool cacheless) + { + var path = SeedRenamed(listsNarrow: false, alsoLists: NarrowShortRegistry.GetRequired().SchemaId, name: "tests.restamp_note"); + var before = Fingerprint(path); + + Assert.Throws(() => WriteNarrow(path, cacheless, NarrowShortRegistry)); + Assert.That(Fingerprint(path), Is.EqualTo(before)); + } + + [TestCase(false)] + [TestCase(true)] + public void ACachelessWriterOwnsWhatItsRegistryOwns(bool cacheless) + { + var path = SeedRenamed(listsNarrow: false, alsoLists: NarrowShortRegistry.GetRequired().SchemaId, name: "tests.restamp_note"); + + WriteNarrow(path, cacheless, NarrowRegistry); + + Assert.That(Snapshot(path).Records.Single(record => record.Key == "note").SchemaId, Is.EqualTo(NarrowRegistry.GetRequired().SchemaId)); + } + + private static readonly CultDocumentRegistry NarrowListerRegistry = CultDocumentRegistry.ForTypes(new[] { typeof(NoteNarrow), typeof(NoteLister) }); + + // A store whose Narrow-shaped record sits under an id another registered type (Lister) claims: its old id, which its entry + // lists, or its current id. + private string SeedListerRecord(bool underCurrentId) + { + var path = SeedNarrow(directory: false); + var snapshot = Snapshot(path); + var listerId = NarrowListerRegistry.GetRequired().SchemaId; + var entry = snapshot.SchemaCatalog.Single(); + entry.SchemaId = entry.ContentHash = underCurrentId ? listerId : "tests.restamp.lister_published"; + entry.SchemaName = "tests.restamp_lister"; + entry.SchemaVersion = "tests.restamp_lister.v1"; + entry.CompatibleSchemaIds = underCurrentId ? new[] { listerId } : new[] { "tests.restamp.lister_published", ListerOldId }; + foreach (var record in snapshot.Records) + record.SchemaId = underCurrentId ? listerId : ListerOldId; + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + return path; + } + + // A cacheless write owns a record only through a claimant it writes: another registered type that lists the record's id, or + // owns it, and reads it whole keeps the record, whatever the lent registry (or Shared, with none) registers. + // Registry null is Shared, which holds every [CultDocument] type of the test assembly. + [TestCase(false, true)] + [TestCase(false, false)] + [TestCase(true, true)] + [TestCase(true, false)] + public void ACachelessWriterNeverOwnsARecordAnotherRegisteredTypeClaims(bool underCurrentId, bool lendRegistry) + { + var path = SeedListerRecord(underCurrentId); + var before = Fingerprint(path); + + Assert.Throws(() => WriteNarrow(path, cacheless: true, lendRegistry ? NarrowListerRegistry : CultDocumentRegistry.Shared)); + Assert.That(Fingerprint(path), Is.EqualTo(before)); + } + + // Claiming is by entry, not by schema name: a record under an id a same-named other version of the written schema lists is + // that version's, and a write of Narrow does not take it. + [Test] + public void ACachelessWriterNeverOwnsARecordASameNamedVersionClaims() + { + var path = SeedRenamed(listsNarrow: false, alsoLists: WideId); + var snapshot = Snapshot(path); + foreach (var record in snapshot.Records) + record.SchemaId = WideId; + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + var before = Fingerprint(path); + + Assert.Throws(() => WriteNarrow(path, cacheless: true, CultDocumentRegistry.ForTypes(new[] { typeof(NoteNarrow), typeof(NoteGrownDeclaring) }))); + Assert.That(Fingerprint(path), Is.EqualTo(before)); + } + + private const string ListerDriftId = "tests.restamp.lister_drift"; + + // The Lister type's entry as a caller passes it: its id's own listing, or the id list the caller chooses. + private static CultSchemaCatalogEntry ListerEntry(string[] compatible, CultSchemaMemberCatalogEntry[]? members = null, string hashPrefix = "") + { + var full = NarrowListerRegistry.GetRequired().ToCatalogEntry(); + return new CultSchemaCatalogEntry + { + SchemaId = full.SchemaId, + ContentHash = hashPrefix + full.ContentHash, + SchemaName = full.SchemaName, + SchemaVersion = full.SchemaVersion, + CompatibleSchemaIds = compatible, + Members = members ?? full.Members, + CanonicalSchemaJson = full.CanonicalSchemaJson, + }; + } + + private static CultPersistedRecord ListerRecord(byte[]? payload = null) => new() + { + Key = "note", + SchemaId = NarrowListerRegistry.GetRequired().SchemaId, + StoredAt = "2026-10-22T00:00:00.0000000+00:00", + Payload = payload ?? MessagePackSerializer.Serialize(new NoteLister { Name = "note", Value = 2 }), + }; + + // The one stored entry and its records renamed to the given id, name and compatible ids (ContentHash kept), Narrow-shaped. + private string SeedListerStore(string id, string name, params string[] compatible) + { + var path = SeedRenamed(listsNarrow: false, name: name); + var snapshot = Snapshot(path); + var entry = snapshot.SchemaCatalog.Single(); + entry.SchemaId = id; + entry.CompatibleSchemaIds = compatible; + foreach (var seeded in snapshot.Records) + seeded.SchemaId = id; + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + return path; + } + + // A direct WriteRecord is bound by the entry it passes, and by it alone: the entry the registry holds for the same schema id + // changes nothing, lent or not. An entry omitting the old id refuses the record; one listing it owns the record, as a cache would. + [TestCase(false, false)] + [TestCase(false, true)] + [TestCase(true, false)] + [TestCase(true, true)] + public void ThePassedEntryAloneDecidesTheWrittenTypesIds(bool entryListsOldId, bool lend) + { + var path = SeedListerStore(ListerOldId, "tests.restamp_old", ListerOldId); + var passed = ListerEntry(entryListsOldId ? new[] { NarrowListerRegistry.GetRequired().SchemaId, ListerOldId } : new[] { NarrowListerRegistry.GetRequired().SchemaId }); + var registry = lend ? CultDocumentRegistry.ForTypes(new[] { typeof(NoteLister) }) : null; + var before = Fingerprint(path); + + if (entryListsOldId) + { + new SingleFileMessagePackBackingStore(path).WriteRecord(ListerRecord(), passed, null, registry); + Assert.That(Snapshot(path).Records.Single(stored => stored.Key == "note").SchemaId, Is.EqualTo(passed.SchemaId)); + return; + } + Assert.Throws(() => new SingleFileMessagePackBackingStore(path).WriteRecord(ListerRecord(), passed, null, registry)); + Assert.That(Fingerprint(path), Is.EqualTo(before)); + } + + // The written type's own drift, read whole, is owned however the entry lists ids and whatever registry is lent: a registry + // that holds the written type never refuses what its own entry would own. + [TestCase(false, false)] + [TestCase(false, true)] + [TestCase(true, false)] + [TestCase(true, true)] + public void ALentRegistryNeverRefusesTheWrittenTypesOwnDrift(bool passedListsOwnId, bool lend) + { + var path = SeedListerStore(ListerDriftId, "tests.restamp_lister", ListerDriftId); + var own = NarrowListerRegistry.GetRequired().SchemaId; + var passed = ListerEntry(passedListsOwnId ? new[] { own } : Array.Empty()); + + new SingleFileMessagePackBackingStore(path).WriteRecord(ListerRecord(), passed, null, lend ? NarrowListerRegistry : null); + + Assert.That(Snapshot(path).Records.Single(stored => stored.Key == "note").SchemaId, Is.EqualTo(own)); + } + + // A lent registry never widens the shape of the passed entry: an entry narrower than the stored one reads it only by dropping + // a slot, and the registry's wider entry for the same id is not asked. + [TestCase(false)] + [TestCase(true)] + public void ALentRegistryNeverWidensTheShapeOfThePassedEntry(bool lend) + { + var path = SeedListerStore(ListerDriftId, "tests.restamp_lister", ListerDriftId); + var full = NarrowListerRegistry.GetRequired().ToCatalogEntry(); + var passed = ListerEntry(full.CompatibleSchemaIds, full.Members.Take(1).ToArray(), "narrow-"); + var before = Fingerprint(path); + + Assert.Throws(() => + new SingleFileMessagePackBackingStore(path).WriteRecord(ListerRecord(MessagePackSerializer.Serialize(new[] { "note" })), passed, null, lend ? NarrowListerRegistry : null)); + Assert.That(Fingerprint(path), Is.EqualTo(before)); + } + + // An unattached store's Delete then PushAll is a cacheless write that writes no entry, so nothing is declared and no written + // entry can claim the stored record: the removal is refused and the file left as it was. A call that writes nothing owns nothing. + [Test] + public void AnUnattachedStoresRemovalWithNoWrittenEntryOwnsNothing() + { + var path = SeedNarrow(directory: false); + var before = Fingerprint(path); + var descriptor = NarrowRegistry.GetRequired(); + var store = new SingleFileMessagePackBackingStore(path); + + store.Delete(new CultStoredDocument(Note, "2026-10-22T00:00:00.0000000+00:00", descriptor, new NoteNarrow { Name = "note", Value = 1 })); + + Assert.Throws(() => store.PushAll()); + Assert.That(Fingerprint(path), Is.EqualTo(before)); + } + + // The shadow, where it is decided: a written entry answers for its schema id whatever the registry holds, and the registry + // still adds its other types. + [Test] + public void AWrittenEntryShadowsTheLentRegistrysTypeForItsId() + { + var written = NarrowRegistry.GetRequired().ToCatalogEntry(); + var lookup = new CatalogEntryLookup(new[] { written }, NarrowListerRegistry); + var lister = NarrowListerRegistry.GetRequired(); + + Assert.That(lookup.Owner(written.SchemaId), Is.SameAs(written)); + Assert.That(lookup.Named(written.SchemaName), Is.EqualTo(new[] { written }).Using((x, y) => ReferenceEquals(x, y))); + Assert.That(lookup.Listing(written.SchemaId), Is.EqualTo(new[] { written }).Using((x, y) => ReferenceEquals(x, y))); + Assert.That(lookup.Owner(lister.SchemaId)!.SchemaId, Is.EqualTo(lister.SchemaId)); + Assert.That(lookup.Listing(ListerOldId).Select(entry => entry.SchemaId), Is.EqualTo(new[] { lister.SchemaId })); + Assert.That(lookup.Named(lister.SchemaName).Select(entry => entry.SchemaId), Is.EqualTo(new[] { lister.SchemaId })); + + // A write of several entries shadows by the entry that has the id, not by all of them having it. + var other = CultDocumentRegistry.ForTypes(new[] { typeof(NoteOther) }).GetRequired().ToCatalogEntry(); + var several = new CatalogEntryLookup(new[] { other, written }, NarrowListerRegistry); + Assert.That(several.Named(written.SchemaName), Is.EqualTo(new[] { written }).Using((x, y) => ReferenceEquals(x, y))); + } + + // The registry's other types change nothing about a record the written type claims itself. + [Test] + public void AnotherRegisteredTypeDoesNotTakeAWritersOwnDriftRecord() + { + var path = SeedRenamed(listsNarrow: true); + + WriteNarrow(path, cacheless: true, NarrowListerRegistry); + + Assert.That(Snapshot(path).Records.Single(record => record.Key == "note").SchemaId, Is.EqualTo(NarrowRegistry.GetRequired().SchemaId)); + } + + // What the lent registry registers beyond the written type can only make the claim more ambiguous, never own more: a second + // version of the written schema's name refuses a record the write owned by name alone. + [Test] + public void ASecondVersionOfTheWrittenNameInTheLentRegistryOnlyRefuses() + { + var path = SeedRenamed(listsNarrow: false, name: "tests.restamp_note"); + var before = Fingerprint(path); + + Assert.Throws(() => WriteNarrow(path, cacheless: true, CultDocumentRegistry.ForTypes(new[] { typeof(NoteNarrow), typeof(NoteGrown) }))); + Assert.That(Fingerprint(path), Is.EqualTo(before)); + } + + // A record stored under an id its catalog entry only lists (no entry owns it) is published by that entry, and the type that + // reads the entry whole owns the record: a cache and a cacheless writer alike restamp it. + [TestCase(false)] + [TestCase(true)] + public void ARecordUnderAnIdOnlyListedIsPublishedByTheEntryListingIt(bool cacheless) + { + const string listedOnly = "tests.restamp.listedonly"; + var path = SeedRenamed(listsNarrow: true, alsoLists: listedOnly); + var snapshot = Snapshot(path); + foreach (var record in snapshot.Records) + record.SchemaId = listedOnly; + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + + WriteNarrow(path, cacheless); + + Assert.That(Snapshot(path).Records.Single(record => record.Key == "note").SchemaId, Is.EqualTo(NarrowRegistry.GetRequired().SchemaId)); + } + + // A raw record write names no registry, so its one entry is the claimant: it reads a renamed entry that lists its id, or has + // its schema name, and nothing else. + [TestCase(true, "tests.restamp_note_renamed", true)] + [TestCase(false, "tests.restamp_note", true)] + [TestCase(false, "tests.restamp_note_renamed", false)] + public void ARecordWriteWithNoRegistryTakesItsOneEntryAsClaimant(bool listsNarrow, string persistedName, bool owned) + { + var record = Snapshot(SeedNarrow(directory: false)).Records.Single(candidate => candidate.Key == "note"); + var path = SeedRenamed(listsNarrow, name: persistedName); + var before = Fingerprint(path); + var write = new SingleFileMessagePackBackingStore(path); + var entry = NarrowRegistry.GetRequired().ToCatalogEntry(); + + if (owned) + { + write.WriteRecord(record, entry); + Assert.That(Snapshot(path).Records.Single(candidate => candidate.Key == "note").SchemaId, Is.EqualTo(entry.SchemaId)); + } + else + { + Assert.Throws(() => write.WriteRecord(record, entry)); + Assert.That(Fingerprint(path), Is.EqualTo(before)); + } + } + + // An id no catalog entry publishes belongs to nobody a writer can name, so no writer owns it. A store file cannot hold such a + // record (ReadStore refuses it), so the rule is pinned where it is decided, for a cacheless writer's lookup. + [Test] + public void AnUnpublishedIdIsOwnedByNoOne() + { + var narrow = NarrowRegistry.GetRequired().ToCatalogEntry(); + var writes = new[] { narrow }; + Func declares = id => id == narrow.SchemaId; + + Assert.That(CultDocumentRegistry.Owns("tests.restamp.unpublished", writes, declares, new CatalogEntryLookup(writes, null), entry => entry, _ => true), Is.False); + Assert.That(CultDocumentRegistry.Owns(narrow.SchemaId, writes, declares, new CatalogEntryLookup(writes, null), entry => entry, _ => true), Is.True); + } + + private static string VectorPath(string name) + { + for (var dir = new DirectoryInfo(TestContext.CurrentContext.TestDirectory); dir != null; dir = dir.Parent) + { + var vectors = Path.Combine(dir.FullName, "tests", "vectors", "ref-list-drift"); + if (Directory.Exists(vectors)) return Path.Combine(vectors, name); + } + + throw new DirectoryNotFoundException($"tests/vectors/ref-list-drift not found above {TestContext.CurrentContext.TestDirectory}."); + } + + // The store the 1.0.60 writer wrote: its 'list' holder compares compatible with RefListHolder, gaining reference semantics + // on a slot, and ignores no persisted slot, so RefListHolder reads it whole and may write it. + [Test] + public async Task TheCommitted1060VectorsListRecordIsWrittenUnderTheHoldersOwnId() + { + var path = Path.Combine(_directory, "vector.cc"); + File.Copy(VectorPath("v1-0-60-written.cc"), path); + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(RefListDriftTests.RefListTarget), typeof(RefListDriftTests.RefListHolder) }); + var holderId = registry.GetRequired().SchemaId; + var before = Snapshot(path); + Assert.That(before.Records.Single(record => record.Key == "list").SchemaId, Is.Not.EqualTo(holderId), "the vector's list is under an older id"); + + using (var cache = new CultCache(registry)) + { + cache.AddBackingStore(new SingleFileMessagePackBackingStore(path)); + await cache.PullAllBackingStoresAsync(); + var holder = cache.GetAll().Single(); + Assert.That(cache.Commit(batch => batch.Upsert( + new RefListDriftTests.RefListHolder { Name = holder.Name, Items = holder.Items.Reverse().ToArray() }, + new CultRecordHandle(new CultRecordKey("list")))), Is.True); + } + + var after = Snapshot(path); + Assert.That(after.Records.Single(record => record.Key == "list").SchemaId, Is.EqualTo(holderId)); + Assert.That(after.SchemaCatalog.Single(entry => entry.SchemaId == holderId).Members.Single(member => member.Slot == 1).IsReference, Is.True); + foreach (var kept in new[] { "alpha", "beta" }) + { + Assert.That(after.Records.Single(record => record.Key == kept).SchemaId, Is.EqualTo(before.Records.Single(record => record.Key == kept).SchemaId), kept + " keeps its persisted id"); + Assert.That(after.Records.Single(record => record.Key == kept).Payload, Is.EqualTo(before.Records.Single(record => record.Key == kept).Payload), kept + " keeps its bytes"); + } + + using var reopened = new CultCache(registry); + reopened.AddBackingStore(new SingleFileMessagePackBackingStore(path)); + await reopened.PullAllBackingStoresAsync(); + Assert.That(reopened.GetAll().Single().Items.Select(item => item.Key.Value).ToArray(), Is.EqualTo(new[] { "beta", "alpha" })); + } + + private static CultPersistedStoreSnapshot Snapshot(string path) => + CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + } +} diff --git a/tests/GameCult.Caching.Tests/StoreReadabilityTests.cs b/tests/GameCult.Caching.Tests/StoreReadabilityTests.cs new file mode 100644 index 00000000..4aef72a1 --- /dev/null +++ b/tests/GameCult.Caching.Tests/StoreReadabilityTests.cs @@ -0,0 +1,693 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using GameCult.Caching.MessagePack; +using MessagePack; +using NUnit.Framework; +using static GameCult.Caching.Tests.ElementIdTests; + +namespace GameCult.Caching.Tests +{ + // A file is replaced by a flush or a commit exactly when this runtime's own reader opens it: one verdict per file, asked by + // open, flush and commit alike. The bytes and the verdict of every runtime are shared: tests/vectors/document-variants-c2a/readability. + public class StoreReadabilityTests + { + private const int CSharp = 1; + private static readonly CultDocumentRegistry Registry = CultDocumentRegistry.ForTypes(new[] { typeof(IdDeck), typeof(VectorItem) }); + private string _directory = ""; + + [SetUp] + public void SetUp() + { + _directory = Path.Combine(Path.GetTempPath(), "cultlib-readability-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(_directory); + } + + [TearDown] + public void TearDown() + { + if (!Directory.Exists(_directory)) + return; + // A recursive delete cannot remove a link whose target is gone; take the links out by themselves first. + foreach (var entry in Directory.EnumerateFileSystemEntries(_directory).Where(entry => IsALink(entry))) + { + if ((new FileInfo(entry).Attributes & FileAttributes.Directory) != 0) + Directory.Delete(entry, false); + else + File.Delete(entry); + } + + Directory.Delete(_directory, true); + } + + private static string VectorRoot() + { + for (var dir = new DirectoryInfo(TestContext.CurrentContext.TestDirectory); dir != null; dir = dir.Parent) + { + var candidate = Path.Combine(dir.FullName, "tests", "vectors", "document-variants-c2a", "readability"); + if (Directory.Exists(candidate)) + return candidate; + } + + throw new DirectoryNotFoundException($"Shared readability vectors not found above {TestContext.CurrentContext.TestDirectory}."); + } + + public static IEnumerable Vectors() => + File.ReadAllLines(Path.Combine(VectorRoot(), "manifest.txt")) + .Where(line => line.Length > 0 && line[0] != '#') + .Select(line => line.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries)) + .Select(cells => new TestCaseData(cells[0], cells[CSharp] == "reads").SetName($"{cells[0]} {cells[CSharp]}")); + + private static bool HoldsAVariant(byte[] bytes) => + CultDocumentMessagePackSerialization.DeserializeSnapshot(bytes).Records.Any(record => record.Variant != null); + + private static string Header(string path) => CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)).FormatVersion; + + private string Seed(string name) + { + var path = Path.Combine(_directory, name); + using var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry }); + cache.Commit(batch => batch.Upsert(typeof(IdDeck), new IdDeck { Name = "d" }, new CultRecordKey("d"))); + return path; + } + + [TestCaseSource(nameof(Vectors))] + public void OpenReadsExactlyTheFilesTheVectorsSayItReads(string vector, bool reads) + { + var path = Path.Combine(_directory, "open.cc"); + File.WriteAllBytes(path, File.ReadAllBytes(Path.Combine(VectorRoot(), vector))); + + void Open() => CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry }).Dispose(); + + if (reads) + Assert.DoesNotThrow(Open); + else + Assert.That(Open, Throws.TypeOf().With.Property(nameof(CultStoreUnreadableException.Path)).EqualTo(path)); + } + + [TestCaseSource(nameof(Vectors))] + public void AFlushReplacesAFileExactlyWhenItReads(string vector, bool reads) + { + var path = Seed("flush.cc"); + using var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry }); + var bytes = File.ReadAllBytes(Path.Combine(VectorRoot(), vector)); + File.WriteAllBytes(path, bytes); + var store = cache.BackingStores[0]; + + if (reads && HoldsAVariant(bytes)) + { + // The cache has read nothing of this file, and a variant resolves against records it has not read. + Assert.That(() => store.PushAll(), Throws.TypeOf()); + Assert.That(File.ReadAllBytes(path), Is.EqualTo(bytes), "a write into a store holding a variant this cache never read changed it"); + return; + } + + if (reads) + { + store.PushAll(); + // A flush applies its staged writes (none) to the file: it cannot show that a record it did not read holds no + // element id, so the file keeps the header it carries. + Assert.That(Header(path), Is.EqualTo(Header(Path.Combine(VectorRoot(), vector)))); + return; + } + + Assert.That(() => store.PushAll(), Throws.TypeOf()); + Assert.That(File.ReadAllBytes(path), Is.EqualTo(bytes), "a file this runtime cannot read was rewritten"); + } + + [TestCaseSource(nameof(Vectors))] + public void AnUnconditionalCommitReplacesAFileExactlyWhenItReads(string vector, bool reads) + { + var path = Seed("commit.cc"); + using var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry }); + var bytes = File.ReadAllBytes(Path.Combine(VectorRoot(), vector)); + File.WriteAllBytes(path, bytes); + + if (reads && HoldsAVariant(bytes)) + { + Assert.That(() => cache.Commit(batch => batch.Upsert(typeof(IdDeck), new IdDeck { Name = "e" }, new CultRecordKey("e"))), Throws.TypeOf()); + Assert.That(File.ReadAllBytes(path), Is.EqualTo(bytes), "a write into a store holding a variant this cache never read changed it"); + return; + } + + if (reads) + { + cache.Commit(batch => batch.Upsert(typeof(IdDeck), new IdDeck { Name = "e" }, new CultRecordKey("e"))); + Assert.That(Header(path), Is.EqualTo(Header(Path.Combine(VectorRoot(), vector))), + "the commit wrote a record without ids and copied the file's records, which it cannot show hold none"); + return; + } + + Assert.That(() => cache.Commit(batch => batch.Upsert(typeof(IdDeck), new IdDeck { Name = "e" }, new CultRecordKey("e"))), Throws.TypeOf()); + Assert.That(File.ReadAllBytes(path), Is.EqualTo(bytes), "a file this runtime cannot read was rewritten"); + } + + private string DirectoryStore(string name) + { + var path = Path.Combine(_directory, name); + using var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry, UseDirectoryStore = true }); + cache.Commit(batch => batch.Upsert(typeof(IdDeck), new IdDeck { Name = "d" }, new CultRecordKey("d"))); + return path; + } + + // The directory store's refusal names the manifest it refused, as the single-file store names its file. + [Test] + public void AnUnreadableDirectoryManifestIsRefusedNamingIt() + { + var path = DirectoryStore("manifest.cc"); + File.WriteAllBytes(path, new byte[] { 0x01 }); + + Assert.That( + () => CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry, UseDirectoryStore = true }), + Throws.TypeOf().With.Property(nameof(CultStoreUnreadableException.Path)).EqualTo(path)); + } + + private static byte[] StoreWithHeader(string header) => + CultDocumentMessagePackSerialization.SerializeSnapshot(new CultPersistedStoreSnapshot + { + FormatVersion = header, + SchemaCatalog = Array.Empty(), + Records = Array.Empty() + }); + + // A refusal shows a stored header only as cultcache.store.v followed by ASCII digits; any other header is described by its + // length, so what the file says never reaches a log. A single-file store and a directory manifest are refused alike. + [TestCase("cultcache.store.v9", true, false)] + [TestCase("cultcache.store.SECRETHDR", false, false)] + [TestCase("cultcache.store.v", false, false)] + [TestCase("cultcache.store.vÙ£", false, false)] + [TestCase("cultcache.store.v9", true, true)] + [TestCase("cultcache.store.SECRETHDR", false, true)] + [TestCase("cultcache.store.v", false, true)] + [TestCase("cultcache.store.vÙ£", false, true)] + public void ARefusalEchoesAHeaderOnlyAsStoreVersionDigits(string header, bool echoed, bool directoryStore) + { + var path = Path.Combine(_directory, "header.cc"); + File.WriteAllBytes(path, StoreWithHeader(header)); + + var refusal = Assert.Throws(() => + CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry, UseDirectoryStore = directoryStore }))!; + + // The readable formats are named in the same message, so "echoed" is read from how the header is introduced. + Assert.That(refusal.Message.Contains(" is " + header + ";", StringComparison.Ordinal) || refusal.Message.Contains("format " + header + " is not readable", StringComparison.Ordinal), Is.EqualTo(echoed), refusal.Message); + Assert.That(refusal.Message.Contains("an unrecognised cultcache.store.* header of ", StringComparison.Ordinal), Is.EqualTo(!echoed), refusal.Message); + Assert.That(refusal.Message.Contains("SECRETHDR", StringComparison.Ordinal), Is.False, refusal.Message); + if (!echoed) + Assert.That(refusal.Message.Contains("Ù£", StringComparison.Ordinal), Is.False, refusal.Message); + } + + // Only nothing at the path is an empty store. A link whose target is gone is an I/O error on open and on commit, never an + // empty store and never an untyped missing-file or access failure, and the link is left where it was. + [Test] + public void ADanglingDirectoryLinkIsAnIoErrorOnOpenNotAnEmptyStore() + { + var path = Path.Combine(_directory, "dangling.cc"); + CreateDanglingDirectoryLink(path, Path.Combine(_directory, "unmounted")); + + Assert.That(() => CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry }), Throws.TypeOf()); + Assert.That(IsALink(path), "the link was replaced"); + Assert.That(Directory.Exists(Path.Combine(_directory, "unmounted")), Is.False); + } + + [Test] + public void ACommitOntoADanglingDirectoryLinkIsAnIoErrorAndLeavesTheLink() + { + var path = Path.Combine(_directory, "link.cc"); + using var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry }); + CreateDanglingDirectoryLink(path, Path.Combine(_directory, "unmounted")); + + Assert.That(() => cache.Commit(batch => batch.Upsert(typeof(IdDeck), new IdDeck { Name = "e" }, new CultRecordKey("e"))), Throws.TypeOf()); + Assert.That(IsALink(path), "the link was replaced"); + Assert.That(Directory.Exists(Path.Combine(_directory, "unmounted")), Is.False, "a write went through the link"); + } + + [Test] + public void ADanglingFileLinkIsAnIoErrorOnOpenNotAnEmptyStore() + { + var path = Path.Combine(_directory, "dangling-file.cc"); + CreateFileLinkOrIgnore(path, Path.Combine(_directory, "unmounted", "store.cc")); + + Assert.That(() => CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry }), Throws.TypeOf()); + Assert.That(new FileInfo(path).LinkTarget, Is.Not.Null, "the link was replaced"); + Assert.That(Directory.Exists(Path.Combine(_directory, "unmounted")), Is.False); + } + + [Test] + public void ACommitOntoAFileLinkThatDanglesAfterOpenIsAnIoErrorAndLeavesTheLink() + { + var target = Path.Combine(_directory, "target.cc"); + var path = Path.Combine(_directory, "file-link.cc"); + File.WriteAllBytes(target, File.ReadAllBytes(Path.Combine(VectorRoot(), "empty-array.bin"))); + CreateFileLinkOrIgnore(path, target); + using var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry }); + File.Delete(target); + + Assert.That(() => cache.Commit(batch => batch.Upsert(typeof(IdDeck), new IdDeck { Name = "e" }, new CultRecordKey("e"))), Throws.TypeOf()); + Assert.That(new FileInfo(path).LinkTarget, Is.Not.Null, "the link was replaced"); + Assert.That(File.Exists(target), Is.False, "a write went through the link"); + } + + // A directory at the path is something at the path: it is an I/O error on open and on commit, never an empty store, and + // it is left as it was. + [Test] + public void ADirectoryAtTheStorePathIsAnIoErrorOnOpenAndOnCommitNotAnEmptyStore() + { + var path = Path.Combine(_directory, "directory.cc"); + Directory.CreateDirectory(path); + + Assert.That(() => CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry }), + Throws.TypeOf().With.Message.Contains("directory")); + + var later = Path.Combine(_directory, "later.cc"); + using var cache = CultCacheMessagePack.Create(later, new CultCacheOpenOptions { Registry = Registry }); + Directory.CreateDirectory(later); + Assert.That(() => cache.Commit(batch => batch.Upsert(typeof(IdDeck), new IdDeck { Name = "e" }, new CultRecordKey("e"))), + Throws.TypeOf().With.Message.Contains("directory")); + + Assert.That(Directory.GetFileSystemEntries(path), Is.Empty); + Assert.That(Directory.GetFileSystemEntries(later), Is.Empty); + } + + // A link whose target is gone. A directory junction needs no privilege on Windows, where a file symlink needs one this + // workstation lacks; elsewhere a symlink to a missing directory is the same thing. + private static void CreateDanglingDirectoryLink(string path, string target) + { + if (!OperatingSystem.IsWindows()) + { + File.CreateSymbolicLink(path, target); + return; + } + + Directory.CreateDirectory(target); + var mklink = Process.Start(new ProcessStartInfo("cmd.exe", $"/c mklink /J \"{path}\" \"{target}\"") + { + CreateNoWindow = true, + RedirectStandardOutput = true, + RedirectStandardError = true, + })!; + mklink.WaitForExit(); + Assert.That(mklink.ExitCode, Is.Zero, "mklink /J failed"); + Directory.Delete(target); + } + + private static bool IsALink(string path) + { + var attributes = new FileInfo(path).Attributes; + return (int)attributes != -1 && (attributes & FileAttributes.ReparsePoint) != 0; + } + + private static void CreateFileLinkOrIgnore(string path, string target) + { + try + { + File.CreateSymbolicLink(path, target); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + Assert.Ignore("A file symlink needs a privilege this account lacks: " + ex.Message); + } + } + + // An id one entry owns and a later entry lists as compatible names the entry that owns it. + [TestCase("own-id-over-compatible-v3.bin")] + [TestCase("compatible-before-owner-v3.bin")] + public void AnIdAnEntryOwnsNamesThatEntryNotOneThatListsItAsCompatible(string vector) + { + var path = Path.Combine(_directory, "own-id.cc"); + File.WriteAllBytes(path, File.ReadAllBytes(Path.Combine(VectorRoot(), vector))); + + using var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry }); + Assert.That(cache.Get(new CultRecordKey("alpha")), Is.Not.Null, "the record read as the schema that owns its id"); + } + + // The registry resolves a persisted id to the entry that owns it, not to one listed earlier that lists it as compatible. + [Test] + public void ThePersistedSchemaAnIdNamesIsTheEntryThatOwnsIt() + { + CultSchemaCatalogEntry Entry(string id, string name, params string[] compatible) => new() + { + SchemaId = id, SchemaName = name, SchemaVersion = name + ".v1", ContentHash = id, CompatibleSchemaIds = compatible + }; + var catalog = new[] + { + Entry("x.lists", "tests.element_id_deck", "x.own"), + Entry("x.own", "vectors.item", "x.own") + }; + + Assert.That(Registry.ResolvePersistedSchemaReport("x.own", catalog).LocalSchemaName, Is.EqualTo("vectors.item")); + } + + // The catalog a write leaves is derived from its records: one entry per carried id, the entry that owns the id (a registered + // descriptor over an arrived entry), else one that lists it as compatible, written as chosen. + private static CultSchemaCatalogEntry Entry(string id, string name, string hash, params string[] compatible) => new() + { + SchemaId = id, SchemaName = name, SchemaVersion = name + ".v1", ContentHash = hash, CompatibleSchemaIds = compatible + }; + + private static CultPersistedRecord Rec(string key, string schemaId) => new() { Key = key, SchemaId = schemaId, StoredAt = "t" }; + + [Test] + public void ARegisteredDescriptorWinsOverAnArrivedEntryWithTheSameOwnId() + { + var registered = new[] { Entry("x", "new.name", "fresh", "x") }; + var arrived = new[] { Entry("x", "old.name", "stale", "x", "old") }; + + var derived = CultSchemaCatalogEntry.Derive(new[] { Rec("a", "x") }, registered, arrived).Single(); + Assert.That(derived.SchemaName, Is.EqualTo("new.name")); + Assert.That(derived.ContentHash, Is.EqualTo("fresh")); + Assert.That(derived.CompatibleSchemaIds, Is.EqualTo(new[] { "x" }), "written as chosen: no union with the arrived entry's ids"); + } + + [Test] + public void AnEntryThatOwnsAnIdIsChosenOverOneThatListsItAsCompatibleInEveryOrder() + { + var owner = Entry("y", "owner", "h1", "y"); + var lister = Entry("x", "lister", "h2", "x", "y"); + foreach (var arrived in new[] { new[] { owner, lister }, new[] { lister, owner } }) + { + var derived = CultSchemaCatalogEntry.Derive(new[] { Rec("a", "y") }, Array.Empty(), arrived); + Assert.That(derived.Select(entry => entry.SchemaId), Is.EqualTo(new[] { "y" }), "an entry no record needs is not written"); + } + + // A registered entry publishes only the ids it owns: the caller passes it for the ids its write stages, and an id nothing + // staged sits under is published by the arrived entry that lists it, or by none. + var registeredLister = Entry("r", "registered.lister", "h3", "r", "z"); + var arrivedLister = Entry("a", "arrived.lister", "h4", "a", "z"); + var derivedByArrived = CultSchemaCatalogEntry.Derive(new[] { Rec("a", "z") }, new[] { registeredLister }, new[] { arrivedLister }).Single(); + Assert.That(derivedByArrived.SchemaId, Is.EqualTo("a"), "a registered entry that lists the id does not publish it"); + Assert.Throws(() => + CultSchemaCatalogEntry.Derive(new[] { Rec("a", "z") }, new[] { registeredLister }, Array.Empty())); + } + + // Entries of one tier that tie are taken in one fixed order: the catalog does not depend on the order they arrive in. + [Test] + public void EntriesThatTieAreTakenInOneFixedOrder() + { + // The content hash orders them against the order their compatible ids would give: h1 sorts first, its list sorts last. + var one = Entry("x", "n", "h2", "x"); + var two = Entry("x", "n", "h1", "x", "y"); + foreach (var order in new[] { new[] { one, two }, new[] { two, one } }) + { + Assert.That(CultSchemaCatalogEntry.Derive(new[] { Rec("a", "x") }, order, Array.Empty()).Single().ContentHash, Is.EqualTo("h1")); + Assert.That(CultSchemaCatalogEntry.Derive(new[] { Rec("a", "x") }, Array.Empty(), order).Single().ContentHash, Is.EqualTo("h1")); + } + } + + // A registered descriptor that owns an id keeps it against an arrived entry that owns it too and lists the id another + // record sits under, whichever of the two ids sorts first: the arrived entry's list is not merged in, so the record under + // the listed id is published by no chosen entry. + [TestCase("a", "b")] + [TestCase("b", "a")] + public void AnArrivedEntryThatOwnsARegisteredIdDoesNotReplaceTheDescriptorNorPublishWhatItLists(string ownId, string listedId) + { + var registered = new[] { Entry(ownId, "registered", "fresh", ownId) }; + var arrived = new[] { Entry(ownId, "arrived", "stale", ownId, listedId) }; + var records = new[] { Rec("k1", ownId), Rec("k2", listedId) }; + + var refusal = Assert.Throws(() => CultSchemaCatalogEntry.Derive(records, registered, arrived))!; + Assert.That(refusal.SchemaId, Is.EqualTo(listedId)); + Assert.That(refusal.RecordKey, Is.EqualTo("k2")); + } + + // Ids are taken in sorted order, so which entry survives, and so which id the refusal names, does not depend on the order + // the records arrive in. + [Test] + public void TheRefusalNamesTheSameIdWhateverOrderTheRecordsArriveIn() + { + var arrived = new[] { Entry("x", "n", "h1", "x", "y"), Entry("x", "n", "h2", "x", "z") }; + foreach (var records in new[] { new[] { Rec("r1", "y"), Rec("r2", "z") }, new[] { Rec("r2", "z"), Rec("r1", "y") } }) + { + var refusal = Assert.Throws(() => + CultSchemaCatalogEntry.Derive(records, Array.Empty(), arrived))!; + Assert.That(refusal.SchemaId, Is.EqualTo("y")); + Assert.That(refusal.RecordKey, Is.EqualTo("r1")); + } + } + + [Test] + public void TwoArrivedEntriesThatShareAnOwnIdAndDisagreeOnTheSchemaNameRefuseTheWrite() + { + var arrived = new[] { Entry("x", "first", "h1", "x"), Entry("x", "second", "h2", "x") }; + + var refusal = Assert.Throws(() => + CultSchemaCatalogEntry.Derive(new[] { Rec("b", "x"), Rec("a", "x") }, Array.Empty(), arrived))!; + Assert.That(refusal.SchemaId, Is.EqualTo("x")); + Assert.That(refusal.SchemaNames, Is.EquivalentTo(new[] { "first", "second" })); + Assert.That(refusal.RecordKey, Is.EqualTo("a")); + + // A registered descriptor that owns the id settles the disagreement. + Assert.DoesNotThrow(() => CultSchemaCatalogEntry.Derive(new[] { Rec("a", "x") }, new[] { Entry("x", "first", "h3", "x") }, arrived)); + } + + [Test] + public void ARecordNoChosenEntryPublishesRefusesTheWrite() + { + var refusal = Assert.Throws(() => + CultSchemaCatalogEntry.Derive(new[] { Rec("a", "orphan") }, new[] { Entry("x", "n", "h", "x") }, Array.Empty()))!; + Assert.That(refusal.SchemaId, Is.EqualTo("orphan")); + Assert.That(refusal.RecordKey, Is.EqualTo("a")); + } + + // A file entry that names an older schema under the registered schema's id, with a record kept under an older id it lists. + private string LegacyFile(string name) + { + var path = Seed(name); + var manifest = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + var entry = manifest.SchemaCatalog.Single(); + entry.ContentHash = "stale"; + entry.CompatibleSchemaIds = new[] { entry.SchemaId, "old.id" }; + manifest.Records.Single().SchemaId = "old.id"; + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(manifest)); + return path; + } + + // The cache holds the record d, but a commit onto the file writes only its batch: d stays as the file holds it, under an id + // only the file's entry lists, and the registered descriptor does not. No store writer restamps a record; the write is + // refused and the file left as it was. + [Test] + public void ACommitOntoAFileWhoseCachedRecordSitsUnderAnIdOnlyAnArrivedEntryListsIsRefusedAndTheFileLeftAsItWas() + { + var path = Seed("legacy-conditional.cc"); + var manifest = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + var entry = manifest.SchemaCatalog.Single(); + manifest.Records.Single().SchemaId = "old.id"; + entry.ContentHash = "stale"; + entry.CompatibleSchemaIds = new[] { entry.SchemaId, "old.id" }; + var bytes = CultDocumentMessagePackSerialization.SerializeSnapshot(manifest); + File.WriteAllBytes(path, bytes); + + using var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry }); + Assert.That(cache.Get(new CultRecordKey("d")), Is.Not.Null, "the cache reads the record under the older id"); + var refusal = Assert.Throws(() => cache.Commit(batch => + { + batch.Expect(new CultRecordKey("e"), null); + batch.Upsert(typeof(IdDeck), new IdDeck { Name = "e" }, new CultRecordKey("e")); + }))!; + Assert.That(refusal.SchemaId, Is.EqualTo("old.id")); + Assert.That(refusal.RecordKey, Is.EqualTo("d")); + Assert.That(File.ReadAllBytes(path), Is.EqualTo(bytes)); + } + + // Another writer rewrites the record d under a different id, keeping its storedAt. A cache that read d earlier commits a + // record of its own onto the file: the other writer's d is not the cache's to rewrite, so it is left exactly as written. + [Test] + public void ACommitNeverRewritesARecordAnotherWriterChangedThatKeptItsStoredAt() + { + var path = Seed("lost-update.cc"); + var other = Seed("lost-update-other.cc"); + using (var cache = CultCacheMessagePack.Create(other, new CultCacheOpenOptions { Registry = Registry })) + cache.Commit(batch => batch.Upsert(typeof(IdDeck), new IdDeck { Name = "changed-by-other" }, new CultRecordKey("d"))); + var otherPayload = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(other)).Records.Single().Payload; + + using var reader = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry }); + Assert.That(reader.Get(new CultRecordKey("d"))!.Name, Is.EqualTo("d")); + + var manifest = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + var entry = manifest.SchemaCatalog.Single(); + var record = manifest.Records.Single(); + record.SchemaId = "other.writer.id"; + record.Payload = otherPayload; + entry.CompatibleSchemaIds = new[] { entry.SchemaId, "other.writer.id" }; + var bytes = CultDocumentMessagePackSerialization.SerializeSnapshot(manifest); + File.WriteAllBytes(path, bytes); + + Assert.Throws(() => reader.Commit(batch => + { + batch.Expect(new CultRecordKey("e"), null); + batch.Upsert(typeof(IdDeck), new IdDeck { Name = "e" }, new CultRecordKey("e")); + })); + Assert.That(File.ReadAllBytes(path), Is.EqualTo(bytes), "the other writer's record is on disk as it wrote it"); + } + + // One rule for both commits: an unconditional commit onto the file copies the record under the older id as it is, and is + // refused for the same reason a conditional one is. + [Test] + public void AnUnconditionalCommitOntoAFileWhoseRecordSitsUnderAnIdOnlyAnArrivedEntryListsIsRefusedAndTheFileLeftAsItWas() + { + var path = LegacyFile("legacy-unconditional.cc"); + var bytes = File.ReadAllBytes(path); + using var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry }); + + var refusal = Assert.Throws(() => + cache.Commit(batch => batch.Upsert(typeof(IdDeck), new IdDeck { Name = "e" }, new CultRecordKey("e"))))!; + Assert.That(refusal.SchemaId, Is.EqualTo("old.id")); + Assert.That(refusal.RecordKey, Is.EqualTo("d")); + Assert.That(File.ReadAllBytes(path), Is.EqualTo(bytes)); + } + + // A record the cache does not hold, under an id only an arrived entry publishes while a registered descriptor owns that + // entry's own id, cannot be written readable: the write is refused and the file is left as it was. + [Test] + public void ARecordOnlyAnOverriddenEntryPublishesRefusesTheWriteAndLeavesTheFile() + { + var path = Seed("legacy-orphan.cc"); + using var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry }); + var manifest = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + var entry = manifest.SchemaCatalog.Single(); + entry.SchemaName = "vectors.legacy"; + entry.CompatibleSchemaIds = new[] { entry.SchemaId, "old.id" }; + var existing = manifest.Records.Single(); + manifest.Records = new[] { existing, new CultPersistedRecord { Key = "z", SchemaId = "old.id", StoredAt = "t", Payload = existing.Payload } }; + var bytes = CultDocumentMessagePackSerialization.SerializeSnapshot(manifest); + File.WriteAllBytes(path, bytes); + + var refusal = Assert.Throws(() => cache.Commit(batch => + { + batch.Expect(new CultRecordKey("e"), null); + batch.Upsert(typeof(IdDeck), new IdDeck { Name = "e" }, new CultRecordKey("e")); + }))!; + Assert.That(refusal.SchemaId, Is.EqualTo("old.id")); + Assert.That(refusal.RecordKey, Is.EqualTo("z")); + Assert.That(File.ReadAllBytes(path), Is.EqualTo(bytes)); + } + + // The directory store derives its manifest the same way and restamps nothing: a record kept under an older id only the + // manifest's entry lists is refused, and the manifest is left as it was. + [Test] + public void ADirectoryRewriteWhoseKeptRecordSitsUnderAnIdOnlyTheManifestEntryListsIsRefusedAndTheManifestLeftAsItWas() + { + var path = DirectoryStore("legacy-dir.cc"); + var manifest = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + var entry = manifest.SchemaCatalog.Single(); + entry.ContentHash = "stale"; + entry.CompatibleSchemaIds = new[] { entry.SchemaId, "old.id" }; + manifest.Records.Single().SchemaId = "old.id"; + var bytes = CultDocumentMessagePackSerialization.SerializeSnapshot(manifest); + File.WriteAllBytes(path, bytes); + + using (var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry, UseDirectoryStore = true })) + { + var refusal = Assert.Throws(() => + cache.Commit(batch => batch.Upsert(typeof(IdDeck), new IdDeck { Name = "e" }, new CultRecordKey("e"))))!; + Assert.That(refusal.SchemaId, Is.EqualTo("old.id")); + Assert.That(refusal.RecordKey, Is.EqualTo("d")); + } + + Assert.That(File.ReadAllBytes(path), Is.EqualTo(bytes)); + using var reopened = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry, UseDirectoryStore = true }); + Assert.That(reopened.Get(new CultRecordKey("d")), Is.Not.Null); + } + + // A directory manifest in a format the directory store does not read is refused with the typed exception, naming it. + [Test] + public void ADirectoryManifestInAnotherFormatIsRefusedNamingIt() + { + var path = Path.Combine(_directory, "wrong-format.cc"); + File.WriteAllBytes(path, File.ReadAllBytes(Path.Combine(VectorRoot(), "..", "v3-base.msgpack"))); + + Assert.That( + () => CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry, UseDirectoryStore = true }), + Throws.TypeOf().With.Property(nameof(CultStoreUnreadableException.Path)).EqualTo(path)); + } + + // A commit that lands onto the file as it is keeps the header the file carries: a store already v3 stays v3. + [TestCase("../v3-base.msgpack", "cultcache.store.v3")] + [TestCase("compatible-id-only-v3.bin", "cultcache.store.v3")] + [TestCase("empty-array.bin", "cultcache.store.v1")] + public void AConditionalCommitOntoAReadableFileKeepsItsHeader(string vector, string header) + { + var path = Seed("conditional.cc"); + using var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry }); + File.WriteAllBytes(path, File.ReadAllBytes(Path.Combine(VectorRoot(), vector))); + + Assert.That(cache.Commit(batch => + { + batch.Expect(new CultRecordKey("e"), null); + batch.Upsert(typeof(IdDeck), new IdDeck { Name = "e" }, new CultRecordKey("e")); + }), Is.True); + Assert.That(Header(path), Is.EqualTo(header)); + } + + // A record's version is the SHA-256 of the bytes it is stored as: in a single file the record's own slice of the file, read + // here with the MessagePack reader and not the serializer; in a directory the hash the manifest indexes its page under. + [Test] + public void AVersionIsTheSha256OfTheRecordsStoredBytes([Values] bool directory) + { + var source = File.ReadAllBytes(Path.Combine(VectorRoot(), "..", "v3-base.msgpack")); + var slices = new Dictionary(); + var reader = new MessagePackReader(source); + reader.ReadArrayHeader(); + reader.Skip(); + reader.Skip(); + var count = reader.ReadArrayHeader(); + var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(source); + for (var i = 0; i < count; i++) + { + var start = (int)reader.Consumed; + reader.Skip(); + slices[snapshot.Records[i].Key] = Convert.ToHexString(System.Security.Cryptography.SHA256.HashData(source.AsSpan(start, (int)reader.Consumed - start))).ToLowerInvariant(); + } + + var path = Path.Combine(_directory, "version.cc"); + File.WriteAllBytes(path, source); + if (directory) + { + // The same records, written as a directory store: its versions are the manifest's page hashes. + File.Delete(path); + using (var seed = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry, UseDirectoryStore = true })) + seed.Commit(batch => + { + foreach (var key in slices.Keys) + batch.Upsert(typeof(VectorItem), new VectorItem { Name = key }, new CultRecordKey(key)); + }); + slices = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)).Records + .ToDictionary(record => record.Key, record => Convert.ToHexString(record.Payload).ToLowerInvariant()); + } + + using var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry, UseDirectoryStore = directory }); + + Assert.That(cache.AllStoredDocuments.ToDictionary(stored => stored.Key.Value, stored => stored.StoredVersion), Is.EqualTo(slices)); + Assert.That(slices, Is.Not.Empty); + } + + // Every runtime's tests walk the manifest, so a vector without a row would go untested. + [Test] + public void EveryVectorInTheFolderHasAManifestRow() + { + var rows = Vectors().Select(row => (string)row.Arguments[0]).Where(name => !name.StartsWith("..", StringComparison.Ordinal)).OrderBy(name => name, StringComparer.Ordinal); + var files = Directory.GetFiles(VectorRoot(), "*.bin").Select(Path.GetFileName).OrderBy(name => name, StringComparer.Ordinal); + Assert.That(rows, Is.EqualTo(files)); + } + + // The refusal carries what the reader choked on. + [Test] + public void ARefusalKeepsItsCauseAsTheInnerException() + { + var error = Assert.Throws(() => + CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(Path.Combine(VectorRoot(), "truncated.bin"))))!; + Assert.That(error.InnerException, Is.Not.Null); + } + + // The record type of the shared valid store (v3-base.msgpack: alpha and beta), so a cache can open it. + [CultDocument("vectors.item", "vectors.item.v1")] + [MessagePackObject] + public sealed class VectorItem + { + [Key(0)] [CultName] public string Name { get; set; } = ""; + [Key(1)] public int Count { get; set; } + } + } +} diff --git a/tests/GameCult.Caching.Tests/StoreSlices.cs b/tests/GameCult.Caching.Tests/StoreSlices.cs new file mode 100644 index 00000000..a27f2468 --- /dev/null +++ b/tests/GameCult.Caching.Tests/StoreSlices.cs @@ -0,0 +1,27 @@ +using System; +using System.IO; +using System.Linq; +using GameCult.Caching.MessagePack; + +namespace GameCult.Caching.Tests +{ + // The exact bytes a store file holds for its catalog entries and records. A read snapshot carries none: the store that read the + // file owns them, and these read them as the store's reader does. + internal static class StoreSlices + { + internal static byte[] CatalogEntry(string path, string schemaId) + { + var read = CultDocumentMessagePackSerialization.ReadStore(File.ReadAllBytes(path)); + return read.CatalogBytes[Array.FindIndex(read.Snapshot.SchemaCatalog, entry => entry.SchemaId == schemaId)]; + } + + internal static byte[][] CatalogEntries(string path) => + CultDocumentMessagePackSerialization.ReadStore(File.ReadAllBytes(path)).CatalogBytes; + + internal static byte[] Record(string path, string key) + { + var read = CultDocumentMessagePackSerialization.ReadStore(File.ReadAllBytes(path)); + return read.RecordBytes[Array.FindIndex(read.Snapshot.Records, record => record.Key == key)]; + } + } +} diff --git a/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs b/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs index df368bdd..85d17832 100644 --- a/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs +++ b/tests/GameCult.Caching.Tests/UnionArmGuardTests.cs @@ -4,6 +4,7 @@ using System.Buffers; using System.Collections.Generic; using System.IO; using System.Linq; +using System.Runtime.Serialization; using System.Threading.Tasks; using GameCult.Caching; using GameCult.Caching.MessagePack; @@ -20,7 +21,7 @@ namespace GameCult.Caching.Tests { private const string Canary = "CANARY-9d1e47"; - private static readonly CultDocumentRegistry Registry = CultDocumentRegistry.ForTypes(new[] { typeof(GuardDoc), typeof(ShapeDoc) }); + private static readonly CultDocumentRegistry Registry = CultDocumentRegistry.ForTypes(new[] { typeof(GuardDoc), typeof(ShapeDoc), typeof(StubDoc) }); private delegate void Emit(ref MessagePackWriter writer); @@ -42,16 +43,18 @@ namespace GameCult.Caching.Tests w.WriteArrayHeader(4); w.Write(id); w.Write(content); w.WriteNil(); w.Write(weight); }; - private static Emit UnitArm(int key) => (ref MessagePackWriter w) => { w.WriteArrayHeader(2); w.Write(key); w.WriteArrayHeader(0); }; + // Unit8: slots [id]. + private static Emit UnitArm(int key) => (ref MessagePackWriter w) => { w.WriteArrayHeader(2); w.Write(key); w.WriteArrayHeader(1); w.Write("000000000007"); }; - // Branch: slots [children, next]. + // Branch: slots [children, next, id]. private static Emit BranchArm(int key, Emit next, params Emit[] children) => (ref MessagePackWriter w) => { w.WriteArrayHeader(2); w.Write(key); - w.WriteArrayHeader(2); + w.WriteArrayHeader(3); w.WriteArrayHeader(children.Length); foreach (var child in children) child(ref w); next(ref w); + w.Write("000000000008"); }; private static Emit List(params Emit[] items) => (ref MessagePackWriter w) => @@ -121,22 +124,22 @@ namespace GameCult.Caching.Tests [TestCase(7)] [TestCase(10)] public void UnknownArmInFieldRefused(int key) => - AssertRefused(Doc(LeafArm(key, "r", Canary, 7)), NodeName, key, "3, 5, 8, 9"); + AssertRefused(Doc(LeafArm(key, "000000000005", Canary, 7)), NodeName, key, "3, 5, 8, 9"); [TestCase(0)] [TestCase(4)] [TestCase(10)] public void UnknownArmInListRefused(int key) => - AssertRefused(Doc(LeafArm(3, "r", "ok", 1), List(UnitArm(8), LeafArm(key, "i", Canary, 7))), NodeName, key, "3, 5, 8, 9"); + AssertRefused(Doc(LeafArm(3, "000000000005", "ok", 1), List(UnitArm(8), LeafArm(key, "000000000004", Canary, 7))), NodeName, key, "3, 5, 8, 9"); // Inside an arm's own list, and inside an arm's own field: the union nested under a union. [TestCase(4)] [TestCase(10)] public void UnknownArmNestedInArmRefused(int key) { - AssertRefused(Doc(BranchArm(5, Nil, UnitArm(8), LeafArm(key, "c", Canary, 7))), NodeName, key, "3, 5, 8, 9"); - AssertRefused(Doc(BranchArm(5, LeafArm(key, "n", Canary, 7))), NodeName, key, "3, 5, 8, 9"); - AssertRefused(Doc(BranchArm(5, Nil, BranchArm(5, Nil, BranchArm(5, Nil, LeafArm(key, "deep", Canary, 7))))), NodeName, key, "3, 5, 8, 9"); + AssertRefused(Doc(BranchArm(5, Nil, UnitArm(8), LeafArm(key, "000000000006", Canary, 7))), NodeName, key, "3, 5, 8, 9"); + AssertRefused(Doc(BranchArm(5, LeafArm(key, "000000000003", Canary, 7))), NodeName, key, "3, 5, 8, 9"); + AssertRefused(Doc(BranchArm(5, Nil, BranchArm(5, Nil, BranchArm(5, Nil, LeafArm(key, "000000000002", Canary, 7))))), NodeName, key, "3, 5, 8, 9"); } // A union whose arms are also a union has its own key set: 7 is Outer's (OuterOnly) and not Inner's. @@ -173,9 +176,9 @@ namespace GameCult.Caching.Tests foreach (var doc in new[] { new GuardDoc { Root = new Rogue() }, - new GuardDoc { Items = { new Unit8(), new Rogue() } }, - new GuardDoc { Root = new Branch { Children = { new Rogue() } } }, - new GuardDoc { Root = new Branch { Next = new Rogue() } }, + new GuardDoc { Items = { new Unit8 { Id = "000000000007" }, new Rogue() } }, + new GuardDoc { Root = new Branch { Id = "000000000008", Children = { new Rogue() } } }, + new GuardDoc { Root = new Branch { Id = "000000000008", Next = new Rogue() } }, new GuardDoc { Root = new DerivedLeaf() }, new GuardDoc { Crafted = new Unlisted() }, }) @@ -204,29 +207,30 @@ namespace GameCult.Caching.Tests Id = "d1", Root = new Branch { + Id = "000000000008", Children = { - new Leaf { Id = "a", Content = "x", Weight = 1 }, - new Unit8(), - new Branch { Children = { new Branch { Children = { new Leaf { Id = "deep", Content = "y", Weight = 2 } } } } } + new Leaf { Id = "000000000001", Content = "x", Weight = 1 }, + new Unit8 { Id = "000000000007" }, + new Branch { Id = "000000000008", Children = { new Branch { Id = "000000000008", Children = { new Leaf { Id = "000000000002", Content = "y", Weight = 2 } } } } } }, - Next = new Leaf { Id = "n", Content = "z", Weight = 3 } + Next = new Leaf { Id = "000000000003", Content = "z", Weight = 3 } }, - Items = { new Unit8(), new Leaf { Id = "i", Content = "w", Weight = 4 }, null! }, + Items = { new Unit8 { Id = "000000000007" }, new Leaf { Id = "000000000004", Content = "w", Weight = 4 }, null! }, Crafted = new InnerB { N = 9 }, Any = new InnerA { N = 8 } }; private static byte[] KnownArmBytes() => Doc( - BranchArm(5, LeafArm(3, "n", "z", 3), - LeafArm(3, "a", "x", 1), + BranchArm(5, LeafArm(3, "000000000003", "z", 3), + LeafArm(3, "000000000001", "x", 1), UnitArm(8), - BranchArm(5, Nil, BranchArm(5, Nil, LeafArm(3, "deep", "y", 2)))), - List(UnitArm(8), LeafArm(3, "i", "w", 4), Nil), + BranchArm(5, Nil, BranchArm(5, Nil, LeafArm(3, "000000000002", "y", 2)))), + List(UnitArm(8), LeafArm(3, "000000000004", "w", 4), Nil), InnerArm(1, 9), InnerArm(1, 8)); - // Non-zero-start keys with retired gaps, a unit arm, a union inside an arm four deep: the reference shape, unchanged. + // Non-zero-start keys with retired gaps, an id-only arm, a union inside an arm four deep: the reference shape, unchanged. [Test] public void KnownArmsRoundTripByteIdentical() { @@ -243,15 +247,543 @@ namespace GameCult.Caching.Tests [Test] public void ConsumerResolverStillEncodesArms() { - var bytes = CultDocumentMessagePackSerialization.SerializeUntyped(new GuardDoc { Id = "d1", Root = new StubArm() }, typeof(GuardDoc), Registry); - Assert.That(Convert.ToHexString(bytes), Is.EqualTo(Convert.ToHexString(Doc((ref MessagePackWriter w) => + var bytes = CultDocumentMessagePackSerialization.SerializeUntyped(new StubDoc { Id = "d1", Root = new StubArm() }, typeof(StubDoc), Registry); + Assert.That(Convert.ToHexString(bytes), Is.EqualTo(Convert.ToHexString(Raw((ref MessagePackWriter w) => { + w.WriteArrayHeader(2); w.Write("d1"); w.WriteArrayHeader(2); w.Write(9); w.WriteArrayHeader(1); w.Write("via-resolver"); })))); - var back = (GuardDoc)CultDocumentMessagePackSerialization.DeserializeUntyped(typeof(GuardDoc), bytes, Registry); + var back = (StubDoc)CultDocumentMessagePackSerialization.DeserializeUntyped(typeof(StubDoc), bytes, Registry); Assert.That(back.Root, Is.TypeOf()); } + // A list element is written by its [Key] layout or its id never reaches disk: a consumer resolver that claims the arm of + // a list's union is refused on every path that encodes or decodes the document, naming the arm and the resolver, and + // writes nothing. + [Test] + public void ListElementsClaimedByAResolverAreRefused() + { + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(ClaimDoc) }); + var doc = new ClaimDoc { Id = "c1", Items = { new ClaimArm { N = 4 } } }; + void Refused(TestDelegate attempt) + { + var thrown = Assert.Throws(attempt)!; + Assert.That(thrown.Message, Does.Contain("ClaimArm").And.Contain(nameof(ClaimArmResolver)).And.Contain(nameof(ClaimDoc))); + } + + Refused(() => CultDocumentMessagePackSerialization.SerializeUntyped(doc, typeof(ClaimDoc), registry)); + var held = Raw((ref MessagePackWriter w) => { w.WriteArrayHeader(2); w.Write("c1"); w.WriteArrayHeader(0); }); + Refused(() => CultDocumentMessagePackSerialization.DeserializeUntyped(typeof(ClaimDoc), held, registry)); + Refused(() => CultCacheMessagePack.CreateCodec(registry).Serialize(doc.Items, typeof(List), typeof(ClaimDoc))); + + var scratch = Directory.CreateTempSubdirectory().FullName; + try + { + var path = Path.Combine(scratch, "claim.cc"); + using var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = registry }); + Assert.That(Assert.Catch(() => cache.Commit(batch => batch.Upsert(typeof(ClaimDoc), doc, new CultRecordKey("c1"))))!.ToString(), + Does.Contain(nameof(ClaimArmResolver))); + Assert.That(File.Exists(path), Is.False, "nothing reached disk"); + } + finally + { + Directory.Delete(scratch, true); + } + } + + // A resolver's formatter for the list type of an object list drops the elements' ids as surely as one for the element, + // so the collection type is on the id path (Soul s1 P1). A claim on a value list, even beside an object list, is none. + [Test] + public void AResolverClaimingTheListTypeOfAnObjectListIsRefused() + { + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(LcDoc), typeof(LcValueDoc) }); + var doc = new LcDoc { Id = "l1", Items = { new LcArm { Id = "000000000001", N = 4 } } }; + var held = Raw((ref MessagePackWriter w) => { w.WriteArrayHeader(2); w.Write("l1"); w.WriteArrayHeader(0); }); + foreach (var attempt in new TestDelegate[] + { + () => CultDocumentMessagePackSerialization.SerializeUntyped(doc, typeof(LcDoc), registry), + () => CultDocumentMessagePackSerialization.DeserializeUntyped(typeof(LcDoc), held, registry) + }) + { + var thrown = Assert.Throws(attempt)!; + Assert.That(thrown.Message, Does.Contain(nameof(ListClaimResolver)).And.Contain("List`1").And.Contain(nameof(LcDoc))); + } + + var values = new LcValueDoc + { + Id = "v1", + Items = { new LcOther { Id = "000000000002" } }, + Values = { new LcValue { N = 3 }, new LcValue { N = 5 } }, + Points = { new Pair { A = 1, B = 2 } } + }; + var bytes = CultDocumentMessagePackSerialization.SerializeUntyped(values, typeof(LcValueDoc), registry); + var back = (LcValueDoc)CultDocumentMessagePackSerialization.DeserializeUntyped(typeof(LcValueDoc), bytes, registry); + Assert.That(back.Items.Single().Id, Is.EqualTo("000000000002")); + Assert.That(back.Values.Select(value => value.N), Is.EqualTo(new[] { 3, 5 })); + Assert.That(back.Points.Single(), Is.EqualTo(new Pair { A = 1, B = 2 })); + } + + // A resolver that hands out MessagePack's own formatter for a type writes its [Key] layout: it claims nothing, however it + // got there, and the document encodes with the id present (Soul s1 P2). + [Test] + public void AResolverHandingBackTheStandardFormatterIsNoClaim() + { + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(FbDoc) }); + var doc = new FbDoc { Id = "f1", Items = { new FbArm { Id = "000000000009", N = 7 } } }; + + var bytes = CultDocumentMessagePackSerialization.SerializeUntyped(doc, typeof(FbDoc), registry); + + Assert.That(Convert.ToHexString(bytes), Is.EqualTo(Convert.ToHexString(MessagePackSerializer.Serialize(doc, CultDocumentMessagePackSerialization.Options)))); + var back = (FbDoc)CultDocumentMessagePackSerialization.DeserializeUntyped(typeof(FbDoc), bytes, registry); + Assert.That(back.Items.Single().Id, Is.EqualTo("000000000009")); + } + + // A consumer whose resolver composite ends in StandardResolverAllowPrivate (some models keep private [Key] members) gets + // that resolver's formatter for every keyed type. It writes the [Key] layout plus the private members, so it is no + // claim: the document encodes, the id is on disk, and the private value survives. + [Test] + public void AStandardResolverAllowPrivateFormatterIsNoClaim() + { + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(ApDoc), typeof(ApPlainDoc) }); + var doc = new ApDoc { Id = "a1", Items = { new ApArm { Id = "00000000000a", N = 4 }.WithSecret(11) } }; + + var bytes = CultDocumentMessagePackSerialization.SerializeUntyped(doc, typeof(ApDoc), registry); + + var back = (ApDoc)CultDocumentMessagePackSerialization.DeserializeUntyped(typeof(ApDoc), bytes, registry); + Assert.That(back.Items.Single().Id, Is.EqualTo("00000000000a")); + Assert.That(back.Items.Single().Secret, Is.EqualTo(11), "a private plain-value [Key] member is written"); + + var plain = new ApPlainDoc { Id = "a2", Items = { new ApPlainArm { Id = "00000000000b", N = 5 } } }; + var plainBytes = CultDocumentMessagePackSerialization.SerializeUntyped(plain, typeof(ApPlainDoc), registry); + Assert.That(Convert.ToHexString(plainBytes), Is.EqualTo(Convert.ToHexString(MessagePackSerializer.Serialize(plain, CultDocumentMessagePackSerialization.Options)))); + } + + // The walk reads a [MessagePackObject] type by its keys, but MessagePack writes one that is also an ICollection as a + // collection and drops the keys. The codec compares the two for every type on the id path and refuses the document. + [Test] + public void AKeyedTypeMessagePackWritesAsACollectionIsRefused() + { + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(KCollDoc), typeof(KBagDoc) }); + Assert.That(CultElementIds.Inspect(typeof(KCollDoc)).IdPath, Does.Contain(typeof(KColl))); + + var coll = new KCollDoc { Id = "k1" }; + coll.Items.Add(new FbArm { Id = "00000000000c" }); + var thrown = Assert.Throws(() => CultDocumentMessagePackSerialization.SerializeUntyped(coll, typeof(KCollDoc), registry))!; + Assert.That(thrown.Message, Does.Contain($"writes {nameof(KColl)} as a collection, but the element-id walk reads it by its [Key] layout")); + + // A keyed map is read by key too, so a keyed Dictionary subclass is on the path and refused the same way. + var map = Assert.Throws(() => CultDocumentMessagePackSerialization.OptionsForDocument(typeof(KDictDoc)))!; + Assert.That(map.Message, Does.Contain($"writes {nameof(KDict)} as a collection")); + + var bag = new KBagDoc { Id = "b1", Bag = new KBagE { Items = { new FbArm { Id = "00000000000d" } } } }; + var bytes = CultDocumentMessagePackSerialization.SerializeUntyped(bag, typeof(KBagDoc), registry); + var back = (KBagDoc)CultDocumentMessagePackSerialization.DeserializeUntyped(typeof(KBagDoc), bytes, registry); + Assert.That(back.Bag.Items.Single().Id, Is.EqualTo("00000000000d")); + } + + // Every collection shape the walk reads is one MessagePack writes through a collection formatter, so none is refused. + [Test] + public void EveryCollectionShapeTheWalkReadsIsWrittenByACollectionFormatter() + { + foreach (var holder in new[] + { + typeof(G>), typeof(G), typeof(G), typeof(G>), typeof(G>), + typeof(G>), typeof(G>), typeof(G>), typeof(G>), + typeof(G>), typeof(G>), typeof(G>), + typeof(G>), typeof(G>), + typeof(G>), typeof(G>), typeof(G>>), + typeof(G>), typeof(G>), typeof(G>), + typeof(G>), typeof(G>), + typeof(G>), typeof(G>), + typeof(G>), typeof(G), typeof(G), typeof(G), + }) + { + if (holder != typeof(G)) + Assert.That(CultElementIds.Inspect(holder).IdPath, Is.Not.Empty, holder.ToString()); + Assert.DoesNotThrow(() => CultDocumentMessagePackSerialization.OptionsForDocument(holder), holder.ToString()); + } + } + + // One recogniser decides what the walk owns and the door only verifies it: the same types give no registration problem and a + // passing door. A type formatted by attribute or written by the object resolver is one whole value, off the id path. + [Test] + public void OneRecogniserDecidesAndTheDoorAgrees() + { + foreach (var holder in new[] { typeof(G), typeof(G), typeof(G), typeof(G), typeof(G) }) + { + Assert.That(CultElementIds.Inspect(holder).Problems, Is.Empty, holder.ToString()); + Assert.DoesNotThrow(() => CultDocumentMessagePackSerialization.OptionsForDocument(holder), holder.ToString()); + } + + foreach (var whole in new[] { typeof(G), typeof(G), typeof(G), typeof(G) }) + Assert.That(CultElementIds.Inspect(whole).IdPath, Is.Empty, whole.ToString()); + var ap = CultElementIds.Inspect(typeof(G)).IdPath; + Assert.That(ap, Does.Contain(typeof(ApTrueHold)).And.Contain(typeof(List))); + var seq = new G { Value = new DcSeq { Items = { new FbArm() } } }; + Assert.That(CultElementIds.Plan(seq.GetType(), seq, "r", deterministic: false).Count, Is.EqualTo(0)); + } + + // MessagePack throws for a type, or StandardResolver has no formatter for it (an interface or abstract class that is only + // IEnumerable): such a type cannot be persisted, so the door refuses it with the same typed text as a throwing build. + [Test] + public void ATypeMessagePackCannotBuildIsATypedRefusal() + { + foreach (var (holder, name) in new[] { (typeof(G), nameof(IArms)), (typeof(G), nameof(AbsArms)) }) + { + var thrown = Assert.Throws(() => CultDocumentMessagePackSerialization.OptionsForDocument(holder), holder.ToString())!; + Assert.That(thrown.Message, Does.Contain(name).And.Contain("cannot build a formatter")); + Assert.That(thrown.Message, Does.Contain("StandardResolver has no formatter for it"), "a null formatter says so, without a thrown message"); + } + } + + // A keyed type that enumerates lists is read by key and is on the id path, so the door checks it: MessagePack writes it as a + // collection, which loses the ids the walk would mint. + [Test] + public void AKeyedTypeThatEnumeratesListsIsCheckedByTheDoor() + { + Assert.That(CultElementIds.Inspect(typeof(G)).IdPath, Does.Contain(typeof(KLL))); + var thrown = Assert.Throws(() => CultDocumentMessagePackSerialization.OptionsForDocument(typeof(G)))!; + Assert.That(thrown.Message, Does.Contain($"writes {nameof(KLL)} as a collection")); + } + + // A resolver that throws while asked about an id-path type is a typed refusal, whether it is the AllowPrivate formatter + // the claim check asks about (the claim stands) or a resolver ahead (it fails to build). + [Test] + public void AResolverThatThrowsIsATypedRefusal() + { + var first = Assert.Throws(() => CultDocumentMessagePackSerialization.OptionsForDocument(typeof(DkDoc)))!; + Assert.That(first.Message, Does.Contain(nameof(ListClaimResolver)).And.Contain(nameof(DkEl))); + var second = Assert.Throws(() => CultDocumentMessagePackSerialization.OptionsForDocument(typeof(DkDoc2)))!; + Assert.That(second.Message, Does.Contain(nameof(ListClaimResolver)).And.Contain(nameof(DkEl2)).And.Contain("failed to build").And.Not.Contain("target of an invocation"), + "the resolver's own exception is named, not the reflection wrapper"); + + // A type MessagePack itself cannot build a formatter for (two public members on one key) is refused, naming the type. + var third = Assert.Throws(() => CultDocumentMessagePackSerialization.OptionsForDocument(typeof(DkDoc3)))!; + Assert.That(third.Message, Does.Contain(nameof(DkEl3)).And.Contain("cannot build a formatter").And.Not.Contain("target of an invocation")); + } + + [MessagePackObject] + public sealed class KColl : List + { + [Key(0)] public int N; + } + + [CultDocument("tests.kcoll_doc", "tests.kcoll_doc.v1")] + [MessagePackObject] + public sealed class KCollDoc + { + [Key(0)] [CultName] public string Id = string.Empty; + [Key(1)] public KColl Items = new(); + } + + [MessagePackObject] + public sealed class KDict : Dictionary + { + [Key(0)] public int N; + } + + [CultDocument("tests.kdict_doc", "tests.kdict_doc.v1")] + [MessagePackObject] + public sealed class KDictDoc + { + [Key(0)] [CultName] public string Id = string.Empty; + [Key(1)] public KDict Items = new(); + } + + [MessagePackObject] + public sealed class KBagE : IEnumerable + { + [Key(0)] public List Items = new(); + public IEnumerator GetEnumerator() { yield break; } + System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => GetEnumerator(); + } + + [CultDocument("tests.kbag_doc", "tests.kbag_doc.v1")] + [MessagePackObject] + public sealed class KBagDoc + { + [Key(0)] [CultName] public string Id = string.Empty; + [Key(1)] public KBagE Bag = new(); + } + + [MessagePackObject] public sealed class G { [Key(0)] public T? Value; } + [Union(0, typeof(GuArm))] public interface IGu { } + [MessagePackObject] public sealed class GuArm : IGu { [Key(0)] [CultElementId] public string Id = string.Empty; [Key(1)] public List Items = new(); } + [MessagePackObject] public sealed class GHold { [Key(0)] public List Items = new(); } + [MessagePackObject] public struct GStruct { [Key(0)] public List Items; } + +#pragma warning disable CS0169, CS0414, CS0649 + [MessagePackObject] public sealed class DkEl { [Key(0)] [CultElementId] public string Id = string.Empty; [Key(1)] public int N; [Key(1)] private int shadow; } + [MessagePackObject] public sealed class DkEl2 { [Key(0)] [CultElementId] public string Id = string.Empty; [Key(1)] public int N; [Key(1)] private int shadow; } +#pragma warning restore CS0169, CS0414, CS0649 + [MessagePackObject] public sealed class DkEl3 { [Key(0)] [CultElementId] public string Id = string.Empty; [Key(1)] public int N; [Key(1)] public int M; } + [CultDocument("tests.dk_doc3", "tests.dk_doc3.v1")] + [MessagePackObject] + public sealed class DkDoc3 { [Key(0)] [CultName] public string Id = string.Empty; [Key(1)] public List Items = new(); } + [CultDocument("tests.dk_doc", "tests.dk_doc.v1")] + [MessagePackObject] + public sealed class DkDoc { [Key(0)] [CultName] public string Id = string.Empty; [Key(1)] public List Items = new(); } + [CultDocument("tests.dk_doc2", "tests.dk_doc2.v1")] + [MessagePackObject] + public sealed class DkDoc2 { [Key(0)] [CultName] public string Id = string.Empty; [Key(1)] public List Items = new(); } + + [MessagePackFormatter(typeof(FormattedRolesFormatter))] public sealed class FormattedRoles : List { } + public sealed class FormattedRolesFormatter : IMessagePackFormatter + { + public void Serialize(ref MessagePackWriter writer, FormattedRoles value, MessagePackSerializerOptions options) => writer.WriteNil(); + public FormattedRoles Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) { reader.Skip(); return new FormattedRoles(); } + } + + [DataContract] public sealed class DcSeq : IEnumerable + { + [DataMember(Order = 0)] public List Items = new(); + public IEnumerator GetEnumerator() => Items.GetEnumerator(); + System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => GetEnumerator(); + } + public interface IArms : IEnumerable { } + public abstract class AbsArms : IEnumerable + { + public abstract IEnumerator GetEnumerator(); + System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => GetEnumerator(); + } + [MessagePackObject] public sealed class KLL : List> { [Key(0)] public int N; } +#pragma warning disable CS0169, CS0414, CS0649 + [MessagePackObject(AllowPrivate = true)] public sealed class ApTrueHold { [Key(0)] private List items = new(); } +#pragma warning restore CS0169, CS0414, CS0649 + [DataContract] public sealed class DcColl : List { [DataMember(Order = 0)] public int N; } + [DataContract] public sealed class DcBad : List { [DataMember(Order = 0)] public int A; [DataMember(Order = 0)] public int B; } + + public sealed class DkFormatter : IMessagePackFormatter + { + public void Serialize(ref MessagePackWriter writer, DkEl value, MessagePackSerializerOptions options) => writer.Write(value.N); + public DkEl Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) => new DkEl { N = reader.ReadInt32() }; + } + + [MessagePackObject] + public sealed class ApArm + { + [Key(0)] [CultElementId] public string Id = string.Empty; + [Key(1)] public int N; + [Key(2)] private int secret; + [IgnoreMember] public int Secret => secret; + public ApArm WithSecret(int value) + { + secret = value; + return this; + } + } + + [CultDocument("tests.ap_doc", "tests.ap_doc.v1")] + [MessagePackObject] + public sealed class ApDoc + { + [Key(0)] [CultName] public string Id = string.Empty; + [Key(1)] public List Items = new(); + } + + [MessagePackObject] + public sealed class ApPlainArm + { + [Key(0)] [CultElementId] public string Id = string.Empty; + [Key(1)] public int N; + } + + [CultDocument("tests.ap_plain_doc", "tests.ap_plain_doc.v1")] + [MessagePackObject] + public sealed class ApPlainDoc + { + [Key(0)] [CultName] public string Id = string.Empty; + [Key(1)] public List Items = new(); + } + + [MessagePackObject] + public sealed class LcArm + { + [Key(0)] [CultElementId] public string Id = string.Empty; + [Key(1)] public int N; + } + + [MessagePackObject] + public sealed class LcOther + { + [Key(0)] [CultElementId] public string Id = string.Empty; + } + + public sealed class LcValue + { + public int N; + } + + [CultDocument("tests.lc_doc", "tests.lc_doc.v1")] + [MessagePackObject] + public sealed class LcDoc + { + [Key(0)] [CultName] public string Id = string.Empty; + [Key(1)] public List Items = new(); + } + + [CultDocument("tests.lc_value_doc", "tests.lc_value_doc.v1")] + [MessagePackObject] + public sealed class LcValueDoc + { + [Key(0)] [CultName] public string Id = string.Empty; + [Key(1)] public List Items = new(); + [Key(2)] public List Values = new(); + [Key(3)] public List Points = new(); + } + + [MessagePackObject] + public sealed class FbArm + { + [Key(0)] [CultElementId] public string Id = string.Empty; + [Key(1)] public int N; + } + + [CultDocument("tests.fb_doc", "tests.fb_doc.v1")] + [MessagePackObject] + public sealed class FbDoc + { + [Key(0)] [CultName] public string Id = string.Empty; + [Key(1)] public List Items = new(); + } + + // Claims List (an object list's type: refused), List and List (value lists: accepted, as a math + // resolver claims List), and answers FbArm with MessagePack's own formatter (no claim). + public sealed class ListClaimResolver : IFormatterResolver + { + public static readonly IFormatterResolver Instance = new ListClaimResolver(); + + public IMessagePackFormatter? GetFormatter() + { + if (typeof(T) == typeof(List)) return (IMessagePackFormatter)(object)new NilFormatter(); + if (typeof(T) == typeof(List)) return (IMessagePackFormatter)(object)new ValuesFormatter(); + if (typeof(T) == typeof(List)) return (IMessagePackFormatter)(object)new PointsFormatter(); + if (typeof(T) == typeof(FbArm)) return global::MessagePack.Resolvers.StandardResolver.Instance.GetFormatter(); + if (typeof(T) == typeof(DkEl)) return (IMessagePackFormatter)(object)new DkFormatter(); + if (typeof(T) == typeof(DkEl2)) return global::MessagePack.Resolvers.StandardResolverAllowPrivate.Instance.GetFormatter(); + if (typeof(T) == typeof(ApArm) || typeof(T) == typeof(ApDoc) || typeof(T) == typeof(List) || + typeof(T) == typeof(ApPlainArm) || typeof(T) == typeof(ApPlainDoc) || typeof(T) == typeof(List)) + return global::MessagePack.Resolvers.StandardResolverAllowPrivate.Instance.GetFormatter(); + return null; + } + + private sealed class NilFormatter : IMessagePackFormatter> + { + public void Serialize(ref MessagePackWriter writer, List value, MessagePackSerializerOptions options) => writer.WriteNil(); + + public List Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) + { + reader.Skip(); + return new List(); + } + } + + private sealed class ValuesFormatter : IMessagePackFormatter> + { + public void Serialize(ref MessagePackWriter writer, List value, MessagePackSerializerOptions options) + { + writer.WriteArrayHeader(value.Count); + foreach (var item in value) writer.Write(item.N); + } + + public List Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) + { + var count = reader.ReadArrayHeader(); + var list = new List(count); + for (var index = 0; index < count; index++) list.Add(new LcValue { N = reader.ReadInt32() }); + return list; + } + } + + private sealed class PointsFormatter : IMessagePackFormatter> + { + public void Serialize(ref MessagePackWriter writer, List value, MessagePackSerializerOptions options) + { + writer.WriteArrayHeader(value.Count); + foreach (var item in value) { writer.WriteArrayHeader(2); writer.Write(item.A); writer.Write(item.B); } + } + + public List Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) + { + var count = reader.ReadArrayHeader(); + var list = new List(count); + for (var index = 0; index < count; index++) + { + reader.ReadArrayHeader(); + list.Add(new Pair { A = reader.ReadSingle(), B = reader.ReadSingle() }); + } + + return list; + } + } + } + + // Composed after ListClaimResolver: once that resolver has answered FbArm with the standard formatter the composite never + // asks this one, so its claim on the same type must not refuse the document. + public sealed class FbLaterClaimResolver : IFormatterResolver + { + public static readonly IFormatterResolver Instance = new FbLaterClaimResolver(); + + public IMessagePackFormatter? GetFormatter() => typeof(T) == typeof(FbArm) ? (IMessagePackFormatter)(object)new LaterFormatter() : null; + + private sealed class LaterFormatter : IMessagePackFormatter + { + public void Serialize(ref MessagePackWriter writer, FbArm value, MessagePackSerializerOptions options) => writer.WriteNil(); + + public FbArm Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) + { + reader.Skip(); + return new FbArm(); + } + } + } + + [Test] + public void OptionsRefuseANullDocumentTypeOrAssembly() + { + Assert.Throws(() => CultDocumentMessagePackSerialization.OptionsFor(null!)); + Assert.Throws(() => CultDocumentMessagePackSerialization.OptionsForDocument(null!)); + } + + [Union(0, typeof(ClaimArm))] + public abstract class ClaimNode { } + + [MessagePackObject] + public sealed class ClaimArm : ClaimNode + { + [Key(0)] [CultElementId] public string Id = string.Empty; + [Key(1)] public int N; + } + + [CultDocument("tests.claim_doc", "tests.claim_doc.v1")] + [MessagePackObject] + public sealed class ClaimDoc + { + [Key(0)] [CultName] public string Id = string.Empty; + [Key(1)] public List Items = new(); + } + + public sealed class ClaimArmResolver : IFormatterResolver + { + public static readonly IFormatterResolver Instance = new ClaimArmResolver(); + public IMessagePackFormatter? GetFormatter() => typeof(T) == typeof(ClaimArm) ? (IMessagePackFormatter)(object)new ClaimFormatter() : null; + + private sealed class ClaimFormatter : IMessagePackFormatter + { + public void Serialize(ref MessagePackWriter writer, ClaimArm value, MessagePackSerializerOptions options) + { + writer.WriteArrayHeader(1); + writer.Write(value.N); + } + + public ClaimArm Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) + { + reader.ReadArrayHeader(); + return new ClaimArm { N = reader.ReadInt32() }; + } + } + } + // The guard wraps the composite, so the composite's own order is still the contract: a consumer resolver outranks the // document resolver (CultRecordRef) and MessagePack's own object formatter, here for a type it also knows. [Test] @@ -393,7 +925,11 @@ namespace GameCult.Caching.Tests (shapes ?? List())(ref w); }); - private static Emit CircleArm(int key, int radius) => InnerArm(key, radius); + private static Emit CircleArm(int key, int radius) => (ref MessagePackWriter w) => + { + w.WriteArrayHeader(2); w.Write(key); + w.WriteArrayHeader(2); w.Write(radius); w.Write("0000000000a1"); + }; [TestCase(1)] [TestCase(3)] @@ -426,7 +962,7 @@ namespace GameCult.Caching.Tests [Test] public void KnownArmsOfAnInterfaceUnionByteIdentical() { - var doc = new ShapeDoc { Id = "s1", Shape = new Square { N = 4 }, Shapes = { new Circle { N = 2 }, null! } }; + var doc = new ShapeDoc { Id = "s1", Shape = new Square { N = 4, Id = "0000000000a1" }, Shapes = { new Circle { N = 2, Id = "0000000000a1" }, null! } }; var expected = Shapes(CircleArm(2, 4), List(CircleArm(0, 2), Nil)); Assert.That(Convert.ToHexString(CultDocumentMessagePackSerialization.SerializeUntyped(doc, typeof(ShapeDoc), Registry)), Is.EqualTo(Convert.ToHexString(expected))); Assert.That(Convert.ToHexString(MessagePackSerializer.Serialize(doc, MessagePackSerializerOptions.Standard)), Is.EqualTo(Convert.ToHexString(expected))); @@ -455,13 +991,13 @@ namespace GameCult.Caching.Tests new CultPersistedRecord { Key = "base", SchemaId = descriptor.SchemaId, StoredAt = "2026-01-01T00:00:00.0000000Z", - Payload = Doc(LeafArm(3, "r", "ok", 1)) + Payload = Doc(LeafArm(3, "000000000005", "ok", 1)) }, new CultPersistedRecord { Key = "variant", SchemaId = descriptor.SchemaId, StoredAt = "2026-01-01T00:00:00.0000000Z", Payload = Array.Empty(), - Variant = new CultVariantDelta("base", new[] { CultVariantOverride.Set(1, Raw(LeafArm(4, "r", Canary, 7))) }) + Variant = new CultVariantDelta("base", new[] { CultVariantOverride.Set(1, Raw(LeafArm(4, "000000000005", Canary, 7))) }) } } })); @@ -495,14 +1031,14 @@ namespace GameCult.Caching.Tests try { var path = Path.Combine(folder, "guard.cc"); - var genuine = Doc(LeafArm(3, "r", Canary, 7)); + var genuine = Doc(LeafArm(3, "000000000005", Canary, 7)); using (var writer = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry, UseDirectoryStore = directory, StoreFlushOnDispose = true })) { - await writer.UpsertAsync(typeof(GuardDoc), new GuardDoc { Id = "d1", Root = new Leaf { Id = "r", Content = Canary, Weight = 7 } }, new CultRecordKey("d1")); + await writer.UpsertAsync(typeof(GuardDoc), new GuardDoc { Id = "d1", Root = new Leaf { Id = "000000000005", Content = Canary, Weight = 7 } }, new CultRecordKey("d1")); await writer.FlushAsync(); } - var forged = Doc(LeafArm(key, "r", Canary, 7)); + var forged = Doc(LeafArm(key, "000000000005", Canary, 7)); Assert.That(forged.Length, Is.EqualTo(genuine.Length)); // A directory store names a page by the SHA-256 of its bytes and its manifest commits to that hash, so a forged // page is renamed and the manifest told its new hash: the store then holds a well-formed page the reader must refuse. @@ -561,13 +1097,13 @@ namespace GameCult.Caching.Tests } // An arm is [key, armSlots]; the keys here start at 3, skip 0-2, 4, 6 and 7, and never reuse a retired key. - [Union(3, typeof(Leaf)), Union(5, typeof(Branch)), Union(8, typeof(Unit8)), Union(9, typeof(StubArm))] + [Union(3, typeof(Leaf)), Union(5, typeof(Branch)), Union(8, typeof(Unit8)), Union(9, typeof(Nine))] public abstract class GuardNode { } [MessagePackObject] public class Leaf : GuardNode { - [Key(0)] public string Id = string.Empty; + [Key(0)] [CultElementId] public string Id = string.Empty; [Key(1)] public string Content = string.Empty; [Key(3)] public int Weight; } @@ -580,15 +1116,30 @@ namespace GameCult.Caching.Tests { [Key(0)] public List Children = new(); [Key(1)] public GuardNode? Next; + [Key(2)] [CultElementId] public string Id = string.Empty; } - [MessagePackObject] public sealed class Unit8 : GuardNode { } + [MessagePackObject] public sealed class Unit8 : GuardNode { [Key(0)] [CultElementId] public string Id = string.Empty; } + + [MessagePackObject] public sealed class Nine : GuardNode { [Key(0)] [CultElementId] public string Id = string.Empty; } // A subtype the union does not name. [MessagePackObject] public sealed class Rogue : GuardNode { [Key(0)] public string Note = string.Empty; } - // Written and read by a consumer resolver, not by MessagePack. - public sealed class StubArm : GuardNode { } + // Written and read by a consumer resolver, not by MessagePack. It can only sit outside a list: a list element is written + // by its [Key] layout, or its id never reaches disk (ListElementsClaimedByAResolverAreRefused). + [Union(9, typeof(StubArm))] + public abstract class StubNode { } + + public sealed class StubArm : StubNode { [Key(0)] [CultElementId] public string Id = string.Empty; } + + [CultDocument("tests.stub_doc", "tests.stub_doc.v1")] + [MessagePackObject] + public sealed class StubDoc + { + [Key(0)] [CultName] public string Id = string.Empty; + [Key(1)] public StubNode? Root; + } public sealed class StubArmResolver : IFormatterResolver { @@ -638,8 +1189,8 @@ namespace GameCult.Caching.Tests [Union(0, typeof(Circle)), Union(2, typeof(Square))] public interface IShape { } - [MessagePackObject] public class Circle : IShape { [Key(0)] public int N; } - [MessagePackObject] public sealed class Square : IShape { [Key(0)] public int N; } + [MessagePackObject] public class Circle : IShape { [Key(0)] public int N; [Key(1)] [CultElementId] public string Id = string.Empty; } + [MessagePackObject] public sealed class Square : IShape { [Key(0)] public int N; [Key(1)] [CultElementId] public string Id = string.Empty; } [MessagePackObject] public sealed class Triangle : IShape { } [MessagePackObject] public sealed class BigCircle : Circle { } diff --git a/tests/GameCult.Caching.Tests/UnionParityVectorTests.cs b/tests/GameCult.Caching.Tests/UnionParityVectorTests.cs index 2561b028..9418a680 100644 --- a/tests/GameCult.Caching.Tests/UnionParityVectorTests.cs +++ b/tests/GameCult.Caching.Tests/UnionParityVectorTests.cs @@ -65,7 +65,7 @@ namespace GameCult.Caching.Tests return File.ReadAllBytes(path); } - // ---- the fixture: a recursive union, a unit arm, a union inside a record inside a union, a list of unions ---- + // ---- the fixture: a recursive union, id-only arms, a union inside a record inside a union, a list of unions ---- // Keys 0, 1, 2, 5 and 6: 3 and 4 are retired, so the key map is not sequential. Wrap is the +2 nesting step beside // Box's +3, so any container depth from 4 up is reachable exactly. @@ -76,26 +76,27 @@ namespace GameCult.Caching.Tests public sealed class Wrap : Node { [Key(0)] public Node? Inner; + [Key(1)] [CultElementId] public string Id = string.Empty; } [MessagePackObject] public sealed class Box : Node { - [Key(0)] public string Id = string.Empty; + [Key(0)] [CultElementId] public string Id = string.Empty; [Key(1)] public List Children = new(); } [MessagePackObject] public sealed class Text : Node { - [Key(0)] public string Id = string.Empty; + [Key(0)] [CultElementId] public string Id = string.Empty; [Key(1)] public string Content = string.Empty; } [MessagePackObject] public sealed class Use : Node { - [Key(0)] public string Id = string.Empty; + [Key(0)] [CultElementId] public string Id = string.Empty; [Key(1)] public Composition Composition = new(); [Key(2)] public List Slots = new(); } @@ -104,7 +105,7 @@ namespace GameCult.Caching.Tests [MessagePackObject] public sealed class Image : Node { - [Key(0)] public string Id = string.Empty; + [Key(0)] [CultElementId] public string Id = string.Empty; [Key(1)] public string Uri = string.Empty; [Key(3)] public string Alt = string.Empty; } @@ -117,17 +118,17 @@ namespace GameCult.Caching.Tests [Key(1)] public Node? Body; } - // Floor and Ceiling are unit arms at the ends of the Int32 key domain. + // Floor and Ceiling are id-only arms at the ends of the Int32 key domain. [Union(0, typeof(Pending)), Union(1, typeof(Completed)), Union(2, typeof(Denied)), Union(int.MinValue, typeof(Floor)), Union(int.MaxValue, typeof(Ceiling))] public abstract class Outcome { } - [MessagePackObject] public sealed class Pending : Outcome { [Key(0)] public string Message = string.Empty; } - [MessagePackObject] public sealed class Completed : Outcome { [Key(0)] public int SourceVersion; } - // A unit arm: written [2, []]. - [MessagePackObject] public sealed class Denied : Outcome { } - [MessagePackObject] public sealed class Floor : Outcome { } - [MessagePackObject] public sealed class Ceiling : Outcome { } + [MessagePackObject] public sealed class Pending : Outcome { [Key(0)] public string Message = string.Empty; [Key(1)] [CultElementId] public string Id = string.Empty; } + [MessagePackObject] public sealed class Completed : Outcome { [Key(0)] public int SourceVersion; [Key(1)] [CultElementId] public string Id = string.Empty; } + // The arms of a list carry an id, so Denied, Floor and Ceiling are written [key, [id]], not as unit arms. + [MessagePackObject] public sealed class Denied : Outcome { [Key(0)] [CultElementId] public string Id = string.Empty; } + [MessagePackObject] public sealed class Floor : Outcome { [Key(0)] [CultElementId] public string Id = string.Empty; } + [MessagePackObject] public sealed class Ceiling : Outcome { [Key(0)] [CultElementId] public string Id = string.Empty; } // Item names Tool and Part under keys 1 and 2; CraftedItem, the inner union both derive from, names them under 0 and 1. [Union(0, typeof(Gem)), Union(1, typeof(Tool)), Union(2, typeof(Part))] @@ -161,24 +162,24 @@ namespace GameCult.Caching.Tests Version = 3, Root = root ?? new Box { - Id = "root", + Id = "0000000000c0", Children = { - new Text { Id = "t1", Content = "hello" }, + new Text { Id = "0000000000c1", Content = "hello" }, new Use { - Id = "u1", + Id = "0000000000c2", Composition = new Composition { Name = "comp", - Body = new Box { Id = "inner", Children = { new Text { Id = "deep", Content = "x" }, new Image { Id = "img", Uri = "uri://a", Alt = "alt" } } } + Body = new Box { Id = "0000000000d0", Children = { new Text { Id = "0000000000d1", Content = "x" }, new Image { Id = "0000000000d2", Uri = "uri://a", Alt = "alt" } } } }, - Slots = { new Text { Id = "s1", Content = "slot" }, null!, new Box { Id = "b2" } } + Slots = { new Text { Id = "0000000000e1", Content = "slot" }, null!, new Box { Id = "0000000000e2" } } }, - new Image { Id = "i2", Uri = "uri://b", Alt = "" } + new Image { Id = "0000000000c3", Uri = "uri://b", Alt = "" } } }, - Outcomes = { new Pending { Message = "wait" }, new Completed { SourceVersion = 7 }, new Denied() }, + Outcomes = { new Pending { Message = "wait", Id = "0000000000a1" }, new Completed { SourceVersion = 7, Id = "0000000000a2" }, new Denied { Id = "0000000000a3" } }, Part = new Tool { Name = "hammer", Uses = 12 }, Crafted = new Part { Name = "gear", Count = 4 }, Missing = null @@ -254,13 +255,13 @@ namespace GameCult.Caching.Tests private static byte[] Surface(Emit? root = null, Emit? outcomes = null, Emit? crafted = null, Extra? extra = null, Emit? trailing = null) { extra ??= new Extra(); - root ??= BoxArm("root", Slots( - TextArm("t1", "hello"), - ArmOf(2, Str("u1"), - Plus(extra.Composition, Str("comp"), BoxArm("inner", Slots(TextArm("deep", "x"), ImageArm("img", "uri://a", "alt")))), - Slots(TextArm("s1", "slot", extra.ListElement), Nil, BoxArm("b2", Slots()))), - ImageArm("i2", "uri://b", "")), extra.RootArm); - outcomes ??= Slots(ArmOf(0, Str("wait")), ArmOf(1, Num(7)), ArmOf(2)); + root ??= BoxArm("0000000000c0", Slots( + TextArm("0000000000c1", "hello"), + ArmOf(2, Str("0000000000c2"), + Plus(extra.Composition, Str("comp"), BoxArm("0000000000d0", Slots(TextArm("0000000000d1", "x"), ImageArm("0000000000d2", "uri://a", "alt")))), + Slots(TextArm("0000000000e1", "slot", extra.ListElement), Nil, BoxArm("0000000000e2", Slots()))), + ImageArm("0000000000c3", "uri://b", "")), extra.RootArm); + outcomes ??= Slots(ArmOf(0, Str("wait"), Str("0000000000a1")), ArmOf(1, Num(7), Str("0000000000a2")), ArmOf(2, Str("0000000000a3"))); crafted ??= ArmOf(1, Str("gear"), Num(4)); var slots = new[] { Str("surface"), Num(3), root, outcomes, ArmOf(1, Str("hammer"), Num(12)), crafted, Nil }; return Raw(Slots(trailing != null ? slots.Append(trailing).ToArray() : extra.Record ? slots.Append(ExtraSlot).ToArray() : slots)); @@ -281,9 +282,9 @@ namespace GameCult.Caching.Tests new Case("unknown-arm-field.msgpack", "refused", "Node", 3, "unknown-arm", Surface(root: ArmOf(3, Str("r"), Str(Canary)))), new Case("unknown-arm-list.msgpack", "refused", "Outcome", 7, "unknown-arm", - Surface(outcomes: Slots(ArmOf(0, Str("wait")), ArmOf(7, Str(Canary))))), + Surface(outcomes: Slots(ArmOf(0, Str("wait"), Str("0000000000a1")), ArmOf(7, Str(Canary))))), new Case("unknown-arm-nested.msgpack", "refused", "Node", 9, "unknown-arm", - Surface(root: BoxArm("root", Slots(BoxArm("l1", Slots(BoxArm("l2", Slots(ArmOf(9, Str("deep"), Str(Canary)))))))))), + Surface(root: BoxArm("0000000000c0", Slots(BoxArm("0000000000d3", Slots(BoxArm("0000000000d4", Slots(ArmOf(9, Str("deep"), Str(Canary)))))))))), new Case("inner-union-outer-key.msgpack", "refused", "CraftedItem", 2, "unknown-arm", Surface(crafted: ArmOf(2, Str("gear"), Num(4)))), new Case("arm-arity.msgpack", "refused", "Node", 1, "arity", @@ -327,6 +328,41 @@ namespace GameCult.Caching.Tests return string.Join(" <- ", parts); } + // ---- unit arms: a union no list reaches, so its arms carry no element id and a member-less arm is a unit arm ---- + + [Union(0, typeof(Waiting)), Union(2, typeof(Denied2)), Union(int.MinValue, typeof(Floor2)), Union(int.MaxValue, typeof(Ceiling2))] + public abstract class UnitOutcome { } + + [MessagePackObject] public sealed class Waiting : UnitOutcome { [Key(0)] public string Message = string.Empty; } + [MessagePackObject] public sealed class Denied2 : UnitOutcome { } + [MessagePackObject] public sealed class Floor2 : UnitOutcome { } + [MessagePackObject] public sealed class Ceiling2 : UnitOutcome { } + + private static MessagePackSerializerOptions UnitOptions => CultDocumentMessagePackSerialization.OptionsFor(typeof(UnionSurface).Assembly); + + // The reference writes a member-less arm as [key, []]; the other runtimes read these files and write the same bytes. + [Test] + public void UnitArmVectorsAreWhatTheReferenceWrites() + { + var arms = new (string File, UnitOutcome Value, int Key, string Name)[] + { + ("unit-arm-denied.msgpack", new Denied2(), 2, nameof(Denied2)), + ("unit-arm-floor.msgpack", new Floor2(), int.MinValue, nameof(Floor2)), + ("unit-arm-ceiling.msgpack", new Ceiling2(), int.MaxValue, nameof(Ceiling2)), + }; + foreach (var (file, value, key, name) in arms) + { + var written = MessagePackSerializer.Serialize(value, UnitOptions); + Assert.That(Convert.ToHexString(written), Is.EqualTo(Convert.ToHexString(Raw(ArmOf(key)))), $"{name}: a unit arm is [key, []]"); + Check(file, written); + + var committed = Committed(file); + var decoded = MessagePackSerializer.Deserialize(committed, UnitOptions); + Assert.That(decoded.GetType().Name, Is.EqualTo(name), file); + Assert.That(Convert.ToHexString(MessagePackSerializer.Serialize(decoded, UnitOptions)), Is.EqualTo(Convert.ToHexString(committed)), $"{file} re-encodes byte-identical"); + } + } + // ---- tests ---- [Test] @@ -387,7 +423,7 @@ namespace GameCult.Caching.Tests var inner = (Box)use.Composition.Body!; Assert.Multiple(() => { - Assert.That(value.Outcomes.OfType().Count(), Is.EqualTo(1), "a unit arm"); + Assert.That(value.Outcomes.OfType().Count(), Is.EqualTo(1), "an id-only arm"); Assert.That(root.Children, Has.Count.GreaterThan(1), "a list of unions"); Assert.That(use.Slots.Any(s => s is Box) && use.Slots.Any(s => s is null), Is.True, "a list of unions holding a nested union arm and nil"); Assert.That(inner.Children.OfType().Count(), Is.EqualTo(1), "a union inside a record inside a union, four arms deep"); @@ -460,30 +496,30 @@ namespace GameCult.Caching.Tests Slots(slots)(ref w); }; - private static Emit WithOutcome(Emit element) => Slots(ArmOf(0, Str("wait")), ArmOf(1, Num(7)), ArmOf(2), element); + private static Emit WithOutcome(Emit element) => Slots(ArmOf(0, Str("wait"), Str("0000000000a1")), ArmOf(1, Num(7), Str("0000000000a2")), ArmOf(2, Str("0000000000a3")), element); - private static Emit WrapChain(int wraps, Emit leaf) => wraps == 0 ? leaf : ArmOf(6, WrapChain(wraps - 1, leaf)); + private static Emit WrapChain(int wraps, Emit leaf) => wraps == 0 ? leaf : ArmOf(6, WrapChain(wraps - 1, leaf), Str("0000000000b1")); private static Node WrapChainValue(int wraps, Node leaf) { - for (var i = 0; i < wraps; i++) leaf = new Wrap { Inner = leaf }; + for (var i = 0; i < wraps; i++) leaf = new Wrap { Inner = leaf, Id = "0000000000b1" }; return leaf; } // n arrays nested inside one another around a single string. private static Emit Nested(int n) => n == 1 ? Slots(Str(Canary)) : Slots(Nested(n - 1)); - private static Emit AtLimitLeaf => BoxArm("leaf", Slots()); - private static Emit PastLimitLeaf => TextArm("leaf", Canary); + private static Emit AtLimitLeaf => BoxArm("0000000000f0", Slots()); + private static Emit PastLimitLeaf => TextArm("0000000000f0", Canary); private static IReadOnlyList BoundsCases() => new[] { new Case("arm-key-int32-min.msgpack", "accepted", "Outcome", int.MinValue, "arm-key-int32-min.msgpack", - Surface(outcomes: WithOutcome(ArmOf(int.MinValue)))), + Surface(outcomes: WithOutcome(ArmOf(int.MinValue, Str("0000000000a4"))))), new Case("arm-key-int32-max.msgpack", "accepted", "Outcome", int.MaxValue, "arm-key-int32-max.msgpack", - Surface(outcomes: WithOutcome(ArmOf(int.MaxValue)))), + Surface(outcomes: WithOutcome(ArmOf(int.MaxValue, Str("0000000000a5"))))), new Case("arm-key-wide-coding.msgpack", "accepted", "Outcome", int.MaxValue, "arm-key-int32-max.msgpack", - Surface(outcomes: WithOutcome(RawKey((ref MessagePackWriter w) => w.WriteUInt64(int.MaxValue))))), + Surface(outcomes: WithOutcome(RawKey((ref MessagePackWriter w) => w.WriteUInt64(int.MaxValue), Str("0000000000a5"))))), new Case("arm-key-above-int32.msgpack", "refused", "Outcome", null, "key-out-of-range", Surface(outcomes: WithOutcome(RawKey((ref MessagePackWriter w) => w.Write(0x80000000u))))), new Case("arm-key-below-int32.msgpack", "refused", "Outcome", null, "key-out-of-range", @@ -649,7 +685,7 @@ namespace GameCult.Caching.Tests var payload = Committed("depth-at-limit.msgpack"); Assert.That(DepthOf(payload), Is.EqualTo(Limit), "the vector nests exactly the limit, counted independently"); Assert.That(Convert.ToHexString(SerializeValue(Decode(payload))), Is.EqualTo(Convert.ToHexString(payload)), "read, then written again"); - var built = Constructed(WrapChainValue(AtLimitWraps, new Box { Id = "leaf" })); + var built = Constructed(WrapChainValue(AtLimitWraps, new Box { Id = "0000000000f0" })); Assert.That(Convert.ToHexString(SerializeValue(built)), Is.EqualTo(Convert.ToHexString(payload)), "the constructed value writes the vector"); } @@ -662,7 +698,7 @@ namespace GameCult.Caching.Tests Assert.That(read, Does.Contain(typeof(UnionSurface).FullName).And.Contain($"the limit is {Limit}").And.Contain("refused on read").And.Not.Contain(Canary)); byte[]? written = null; - var write = Chain(Refusal(() => written = SerializeValue(Constructed(WrapChainValue(PastLimitWraps, new Text { Id = "leaf", Content = Canary }))))); + var write = Chain(Refusal(() => written = SerializeValue(Constructed(WrapChainValue(PastLimitWraps, new Text { Id = "0000000000f0", Content = Canary }))))); Assert.That(written, Is.Null, "nothing is returned"); Assert.That(write, Does.Contain(typeof(UnionSurface).FullName).And.Contain($"the limit is {Limit}").And.Contain("refused on write").And.Not.Contain(Canary)); diff --git a/tests/GameCult.Caching.Tests/VariantTests.cs b/tests/GameCult.Caching.Tests/VariantTests.cs index 7d3a80f3..b8c1fb3d 100644 --- a/tests/GameCult.Caching.Tests/VariantTests.cs +++ b/tests/GameCult.Caching.Tests/VariantTests.cs @@ -181,6 +181,14 @@ namespace GameCult.Caching.Tests var seen = a.Get(BigKey)!; EditBase(b, gear => gear.Tags.Add("pierce")); + // Expect tests the variant's stored delta only, and it did not change, so the condition holds and the write goes on: the + // store holds a variant and moved, so it is refused and reloaded as a write conflict. It is not a mismatch (false). + Assert.Throws(() => a.Commit(batch => + { + batch.Expect(BigKey, seen); + batch.Upsert(typeof(VariantOther), new VariantOther { Name = "marker" }, new CultRecordKey("marker")); + })); + seen = a.Get(BigKey)!; Assert.That(a.Commit(batch => { batch.Expect(BigKey, seen); @@ -189,6 +197,7 @@ namespace GameCult.Caching.Tests a.PullAllBackingStoresAsync(); seen = a.Get(BigKey)!; + b.PullAllBackingStoresAsync(); SeedBig(b, power: 26); Assert.That(a.Commit(batch => { @@ -665,34 +674,301 @@ namespace GameCult.Caching.Tests Assert.That(reopened.Get(other), Is.Not.Null); } - // ---- a variant sheds overrides its type lacks when it is flushed, as a plain record sheds the member ---- - + // A store holding a variant is written only from what its writer last read. Another writer adds a variant whose indexed value + // this cache's staged edit of its base now takes: the write is refused whole, the cache reloads what it refused and what moved, + // and the retry is judged by the cache's own rules. [Test] - public void AFlushShedsAnOverrideOfADroppedSlotSoALaterTypeReusingTheSlotDoesNotResurrectIt() + public void AWriteIntoAVariantStoreAnotherWriterChangedIsRefusedReloadedAndJudgedOnRetry([Values] bool viaCommit) { - var path = PathOf("drift.cc"); - var wide = CultDocumentRegistry.ForTypes(new[] { typeof(DriftWide) }); - using (var writer = Open(path, registry: wide)) + var path = PathOf(viaCommit ? "judged-commit.cc" : "judged-flush.cc"); + using var cache = Open(path); + SeedBase(cache); + using (var other = Open(path)) + SeedBig(other); + var edited = Laser(); + edited.Code = "laser big"; + cache.UpsertAsync(typeof(VariantGear), edited, BaseKey).GetAwaiter().GetResult(); + var before = File.ReadAllBytes(path); + + var conflict = Assert.Throws(() => { - writer.Commit(batch => + if (viaCommit) + cache.Commit(batch => batch.Upsert(typeof(VariantGear), Other("n"), new CultRecordKey("n"))); + else + cache.FlushAllBackingStores(); + })!; + + Assert.That(conflict.RecordKeys, Does.Contain(BaseKey.Value)); + Assert.That(conflict.ChangedKeys, Does.Contain(BigKey.Value)); + Assert.That(cache.Get(BaseKey)!.Code, Is.EqualTo("l1"), "the refused edit is not served"); + Assert.That(cache.Get(BigKey), Is.Not.Null, "the moved record is loaded"); + Assert.That(cache.IsDirty, Is.False); + Assert.That(File.ReadAllBytes(path), Is.EqualTo(before), "nothing was written"); + Assert.That(Refused(() => cache.UpsertAsync(typeof(VariantGear), edited, BaseKey).GetAwaiter().GetResult()).Message, + Does.Contain(BigKey.Value), "the retry is judged by the cache's own index rule"); + } + + // A refused base removal leaves the cache serving the store, writable, and judging the next attempt itself. + [Test] + public void ARefusedBaseRemovalReloadsAndLeavesTheCacheWritable() + { + var path = PathOf("gear.cc"); + var n = new CultRecordKey("n"); + using var a = Open(path); + SeedBase(a); + using (var b = Open(path)) + SeedBig(b); + a.Remove(BaseKey); + + var conflict = Assert.Throws(() => a.FlushAllBackingStores())!; + + Assert.That(conflict.RecordKeys, Is.EqualTo(new[] { BaseKey.Value })); + Assert.That(conflict.ChangedKeys, Is.EqualTo(new[] { BigKey.Value })); + Assert.That(a.Get(BaseKey), Is.Not.Null); + Assert.That(a.Get(BigKey), Is.Not.Null); + Assert.That(a.IsDirty, Is.False); + Assert.DoesNotThrow(() => a.FlushAllBackingStores()); + Assert.DoesNotThrow(() => a.Commit(batch => batch.Upsert(typeof(VariantGear), Named("n", "nn"), n))); + Assert.That(Refused(() => a.Remove(BaseKey)).Message, Does.Contain(BigKey.Value), "a base with variants is refused by this cache's own rule"); + + using var reopened = Open(path); + Assert.That(reopened.Get(n), Is.Not.Null); + Assert.That(reopened.Get(BigKey), Is.Not.Null); + Assert.That(reopened.Get(BaseKey), Is.Not.Null); + } + + // A held record another writer changed is not trusted: the write is refused although this cache never staged it. + [Test] + public void AWriteIsRefusedWhenAnotherWriterChangedAVariantThisCacheHolds() + { + var path = PathOf("gear.cc"); + using var a = Open(path); + SeedBase(a); + SeedBig(a); + using (var b = Open(path)) + b.Commit(batch => batch.UpsertVariant(BigKey, BaseKey, new[] { - batch.Upsert(typeof(DriftWide), new DriftWide { Name = "drift", Power = 1, Wing = "base wing" }, new CultRecordKey("drift-base")); - batch.UpsertVariant(new CultRecordKey("drift-variant"), new CultRecordKey("drift-base"), new[] - { - writer.Override(nameof(DriftWide.Name), "drift variant"), - writer.Override(nameof(DriftWide.Wing), "stale wing") - }); + b.Override(nameof(VariantGear.Name), "laser big"), + b.Override(nameof(VariantGear.Power), 99), + b.Override(nameof(VariantGear.Code), "laser big") + })); + Assert.That(a.Get(BigKey)!.Power, Is.EqualTo(25)); + + var conflict = Assert.Throws(() => + a.Commit(batch => batch.Upsert(typeof(VariantGear), Named("y", "yy"), new CultRecordKey("y"))))!; + + Assert.That(conflict.ChangedKeys, Is.EqualTo(new[] { BigKey.Value })); + Assert.That(a.Get(BigKey)!.Power, Is.EqualTo(99), "the cache serves the other writer's override after the refusal"); + } + + // A write that would add a variant is judged like one into a variant store: a plain store another writer changed refuses it. + [Test] + public void AVariantWrittenIntoAPlainStoreAnotherWriterChangedIsRefusedThenLandsOnRetry() + { + var path = PathOf("plain-then-variant.cc"); + var z = new CultRecordKey("z"); + using (var seed = Open(path)) + SeedBase(seed); + using (var a = Open(path)) + { + using (var b = Open(path)) + b.Commit(batch => batch.Upsert(typeof(VariantGear), Other("zc"), z)); + + var conflict = Assert.Throws(() => SeedBig(a))!; + + Assert.That(conflict.RecordKeys, Does.Contain(BigKey.Value)); + Assert.That(conflict.ChangedKeys, Is.EqualTo(new[] { z.Value })); + Assert.That(a.Get(z), Is.Not.Null, "the refusal reloaded the record that moved"); + Assert.DoesNotThrow(() => SeedBig(a)); + } + + using var reopened = Open(path); + Assert.That(reopened.Get(BigKey), Is.Not.Null); + Assert.That(reopened.Get(z), Is.Not.Null); + } + + // A plain write copies another writer's record forward without reading it: this cache has still not read it, so a variant + // written next is refused. + [Test] + public void ARecordACachePlainWriteCopiedForwardIsStillUnreadWhenAVariantIsWrittenNext() + { + var path = PathOf("copied-unread.cc"); + var z = new CultRecordKey("z"); + using (var seed = Open(path)) + SeedBase(seed); + using var a = Open(path); + using (var b = Open(path)) + b.Commit(batch => batch.Upsert(typeof(VariantGear), Other("zc"), z)); + a.Commit(batch => batch.Upsert(typeof(VariantGear), Other("yc"), new CultRecordKey("y"))); + + var conflict = Assert.Throws(() => SeedBig(a))!; + + Assert.That(conflict.ChangedKeys, Is.EqualTo(new[] { z.Value })); + } + + // ---- a write copies what it did not stage, and re-encodes what it stages under the type's own id ---- + + private static readonly string DriftWideId = CultDocumentRegistry.ForTypes(new[] { typeof(DriftWide) }).GetRequired(typeof(DriftWide)).SchemaId; + + // The same schema name as Wide, a later version that lists Wide's id as compatible and has no Wing. + private static readonly Type DriftDeclared = EmittedDocumentTypes.Emit( + "DriftDeclared", + "tests.variant_drift", + "tests.variant_drift.v4", + new[] { new EmittedDocumentTypes.Field("Name", typeof(string), 0, IsName: true), new EmittedDocumentTypes.Field("Power", typeof(int), 1) }, + new[] { DriftWideId }); + + private static readonly CultRecordKey DriftBase = new("drift-base"); + private static readonly CultRecordKey DriftVariant = new("drift-variant"); + + private static CultPersistedStoreSnapshot ReadStore(string path) => + CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + + private void SeedDrift(string path) + { + using var writer = Open(path, registry: CultDocumentRegistry.ForTypes(new[] { typeof(DriftWide) })); + writer.Commit(batch => + { + batch.Upsert(typeof(DriftWide), new DriftWide { Name = "drift", Power = 1, Wing = "base wing" }, DriftBase); + batch.UpsertVariant(DriftVariant, DriftBase, new[] + { + writer.Override(nameof(DriftWide.Name), "drift variant"), + writer.Override(nameof(DriftWide.Wing), "stale wing") + }); + }); + } + + // Narrow reads Wide's records through the schema name and neither owns nor declares Wide's id. A write of another record + // copies them as Wide stored them: bytes, id, storedAt, the override of the slot Narrow lacks, and the entry that publishes + // Wide's id. + [Test] + public void AWriteOfAnotherRecordLeavesAVariantItsTypeHasNotDeclaredAsStored([Values] bool viaFlush) + { + var path = PathOf(viaFlush ? "drift-untouched-flush.cc" : "drift-untouched-commit.cc"); + SeedDrift(path); + var before = ReadStore(path); + var wideEntryBefore = StoreSlices.CatalogEntry(path, DriftWideId); + + using (var narrow = Open(path, registry: CultDocumentRegistry.ForTypes(new[] { typeof(DriftNarrow) }))) + { + Assert.That(narrow.Get(DriftVariant)!.Name, Is.EqualTo("drift variant"), "Narrow reads the variant"); + var other = new CultRecordKey("other"); + if (viaFlush) + { + narrow.UpsertAsync(typeof(DriftNarrow), new DriftNarrow { Name = "other" }, other).GetAwaiter().GetResult(); + narrow.FlushAllBackingStores(); + } + else + narrow.Commit(batch => batch.Upsert(typeof(DriftNarrow), new DriftNarrow { Name = "other" }, other)); + } + + var after = ReadStore(path); + foreach (var key in new[] { DriftBase.Value, DriftVariant.Value }) + { + var was = before.Records.Single(record => record.Key == key); + var is_ = after.Records.Single(record => record.Key == key); + Assert.That((is_.SchemaId, is_.StoredAt, is_.Payload), Is.EqualTo((was.SchemaId, was.StoredAt, was.Payload)), key); + } + + Assert.That(OverriddenSlotsOn(path, DriftVariant.Value).OrderBy(slot => slot), Is.EqualTo(new[] { 0, 2 }), "the dropped slot's override is still stored"); + Assert.That(StoreSlices.CatalogEntry(path, DriftWideId), + Is.EqualTo(wideEntryBefore)); + } + + // A type that lists Wide's id as compatible owns Wide's records: staging one writes it under the type's own id, at a later + // storedAt, without the member the type dropped, so a later type reusing that slot does not resurrect it. + [Test] + public void AStagedWriteOfADeclaredCompatibleRecordIsReEncodedUnderTheTypesOwnIdAndShedsTheDroppedSlot() + { + var path = PathOf("drift-declared.cc"); + SeedDrift(path); + var wideBase = ReadStore(path).Records.Single(record => record.Key == DriftBase.Value); + var ownId = CultDocumentRegistry.ForTypes(new[] { DriftDeclared }).GetRequired(DriftDeclared).SchemaId; + + using (var declared = Open(path, registry: CultDocumentRegistry.ForTypes(new[] { DriftDeclared }))) + { + declared.Commit(batch => + { + batch.Upsert(DriftDeclared, declared.Get(DriftBase)!, DriftBase); + batch.UpsertVariant(DriftVariant, DriftBase, new[] { declared.Override(DriftDeclared, "Name", "drift variant") }); }); } - using (var narrow = Open(path, registry: CultDocumentRegistry.ForTypes(new[] { typeof(DriftNarrow) }))) - narrow.BackingStores[0].PushAll(); - - Assert.That(OverriddenSlotsOn(path, "drift-variant"), Is.EqualTo(new[] { 0 }), "the dropped slot's override is gone from the flushed record"); + var written = ReadStore(path); + Assert.That(written.Records.Select(record => record.SchemaId), Is.All.EqualTo(ownId)); + Assert.That(string.CompareOrdinal(written.Records.Single(record => record.Key == DriftBase.Value).StoredAt, wideBase.StoredAt), Is.GreaterThan(0)); + Assert.That(OverriddenSlotsOn(path, DriftVariant.Value), Is.EqualTo(new[] { 0 }), "the dropped slot's override is gone from the written record"); using var reused = Open(path, registry: CultDocumentRegistry.ForTypes(new[] { typeof(DriftReused) })); - Assert.That(reused.Get(new CultRecordKey("drift-base"))!.Colour, Is.Empty); - Assert.That(reused.Get(new CultRecordKey("drift-variant"))!.Colour, Is.Empty, "the variant inherits its base's colour, not the old wing"); + Assert.That(reused.Get(DriftBase)!.Colour, Is.Empty); + Assert.That(reused.Get(DriftVariant)!.Colour, Is.Empty, "the variant inherits its base's colour, not the old wing"); + } + + public enum UndeclaredWrite + { + UpsertBase, + UpsertVariant, + FlattenVariant, + RemoveBase, + RemoveVariant + } + + // Narrow reads Wide's records through the schema name and does not list Wide's id, so they are read-only to it: a staged write, + // flatten or removal of either is refused typed, and the store keeps them as Wide stored them, the dropped slot's override too. + [Test] + public void AnUndeclaredDroppedSlotRecordThatIsWrittenIsRefused([Values] UndeclaredWrite write) + { + var path = PathOf("drift-undeclared-" + write + ".cc"); + SeedDrift(path); + var before = File.ReadAllBytes(path); + + using (var narrow = Open(path, registry: CultDocumentRegistry.ForTypes(new[] { typeof(DriftNarrow) }))) + { + var key = write is UndeclaredWrite.UpsertBase or UndeclaredWrite.RemoveBase ? DriftBase : DriftVariant; + var refusal = Assert.Throws(() => + { + switch (write) + { + case UndeclaredWrite.UpsertBase: + narrow.Commit(batch => batch.Upsert(typeof(DriftNarrow), new DriftNarrow { Name = "changed" }, DriftBase)); + break; + case UndeclaredWrite.UpsertVariant: + narrow.Commit(batch => batch.UpsertVariant(DriftVariant, DriftBase, new[] { narrow.Override(nameof(DriftNarrow.Name), "changed") })); + break; + case UndeclaredWrite.FlattenVariant: + narrow.FlattenAsync(DriftVariant).GetAwaiter().GetResult(); + break; + case UndeclaredWrite.RemoveBase: + narrow.Commit(batch => batch.Remove(DriftBase)); + break; + case UndeclaredWrite.RemoveVariant: + narrow.Commit(batch => batch.Remove(DriftVariant)); + break; + } + })!; + Assert.That((refusal.RecordKey, refusal.SchemaId), Is.EqualTo((key.Value, DriftWideId))); + narrow.FlushAllBackingStores(); + } + + Assert.That(File.ReadAllBytes(path), Is.EqualTo(before)); + Assert.That(OverriddenSlotsOn(path, DriftVariant.Value).OrderBy(slot => slot), Is.EqualTo(new[] { 0, 2 })); + } + + // Only the read-only key is read-only: a variant of a read-only base, written at a new key, is a write of the variant alone and lands. + [Test] + public void AVariantOfAReadOnlyBaseLands() + { + var path = PathOf("drift-read-only-base.cc"); + SeedDrift(path); + var fresh = new CultRecordKey("fresh-variant"); + var narrowRegistry = CultDocumentRegistry.ForTypes(new[] { typeof(DriftNarrow) }); + + using (var narrow = Open(path, registry: narrowRegistry)) + narrow.Commit(batch => batch.UpsertVariant(fresh, DriftBase, new[] { narrow.Override(nameof(DriftNarrow.Name), "fresh") })); + + using var reopened = Open(path, registry: narrowRegistry); + Assert.That(reopened.Get(fresh)!.Name, Is.EqualTo("fresh")); + Assert.That(ReadStore(path).Records.Single(record => record.Key == DriftBase.Value).SchemaId, Is.EqualTo(DriftWideId)); } [Test] @@ -742,7 +1018,7 @@ namespace GameCult.Caching.Tests } [Test] - public void AnOverrideOfASlotTheTypeNeverHadIsReportedOnLoadAndShedOnTheNextFlush() + public void AnOverrideOfASlotTheTypeNeverHadIsReportedOnEveryLoadAndStaysStoredUntilTheVariantIsWritten() { var path = WriteStore(PlainGear("hand-base"), OverridingRecord("hand-variant", "hand-base", CultVariantOverride.Set(99, new byte[] { 0x01 }), @@ -757,10 +1033,19 @@ namespace GameCult.Caching.Tests cache.BackingStores[0].PushAll(); } - Assert.That(OverriddenSlotsOn(path, "hand-variant").OrderBy(slot => slot), Is.EqualTo(new[] { 0, 2 })); - using var reopened = Open(path); - Assert.That(reopened.BackingStores[0].LastSchemaMigrationReports.SelectMany(report => report.IgnoredExtraSlots), Is.Empty); - Assert.That(reopened.Get(new CultRecordKey("hand-variant"))!.Name, Is.EqualTo("hand variant")); + Assert.That(OverriddenSlotsOn(path, "hand-variant").OrderBy(slot => slot), Is.EqualTo(new[] { 0, 2, 99 }), "a flush that staged nothing copies the variant as stored"); + using (var reopened = Open(path)) + { + Assert.That(reopened.BackingStores[0].LastSchemaMigrationReports.SelectMany(report => report.IgnoredExtraSlots), Is.EqualTo(new[] { 99 })); + Assert.That(reopened.Get(new CultRecordKey("hand-variant"))!.Name, Is.EqualTo("hand variant")); + reopened.Commit(batch => batch.UpsertVariant(new CultRecordKey("hand-variant"), new CultRecordKey("hand-base"), new[] + { + reopened.Override(nameof(VariantGear.Name), "hand variant"), + reopened.Override(nameof(VariantGear.Code), "hand-code") + })); + } + + Assert.That(OverriddenSlotsOn(path, "hand-variant").OrderBy(slot => slot), Is.EqualTo(new[] { 0, 2 }), "a write of the variant stores only what it was given"); } private static int[] OverriddenSlotsOn(string path, string key) => @@ -904,12 +1189,16 @@ namespace GameCult.Caching.Tests a.Override(nameof(VariantGear.Code), "shared") })); - var refused = Refused(() => b.Commit(batch => + void Write() => b.Commit(batch => { batch.Expect(other, null); batch.Upsert(typeof(VariantGear), Named("other", "shared"), other); - })); - Assert.That(refused.Message, Does.Contain(other.Value).And.Contain(BigKey.Value).And.Contain("code").And.Contain("shared")); + }); + + var conflict = Assert.Throws(Write)!; + Assert.That(conflict.ChangedKeys, Does.Contain(BigKey.Value)); + var refused = Refused(Write); + Assert.That(refused.Message, Does.Contain(other.Value).And.Contain(BigKey.Value).And.Contain("code").And.Contain("shared"), "the retry is judged by Resolve"); Assert.That(b.Get(other), Is.Null, "nothing landed in memory"); } @@ -931,12 +1220,16 @@ namespace GameCult.Caching.Tests var seen = b.Get(BaseKey)!; SeedBig(a); - var refused = Refused(() => b.Commit(batch => + void Write() => b.Commit(batch => { batch.Expect(BaseKey, seen); batch.Remove(BaseKey); - })); - Assert.That(refused.Message, Does.Contain(BaseKey.Value).And.Contain(BigKey.Value)); + }); + + var conflict = Assert.Throws(Write)!; + Assert.That(conflict.ChangedKeys, Does.Contain(BigKey.Value)); + var refused = Refused(Write); + Assert.That(refused.Message, Does.Contain(BaseKey.Value).And.Contain(BigKey.Value), "the retry is judged by Resolve"); Assert.That(b.Get(BaseKey), Is.Not.Null, "nothing landed in memory"); } @@ -963,7 +1256,7 @@ namespace GameCult.Caching.Tests using (var w = Open(path)) Assert.That(w.Commit(batch => batch.Upsert(typeof(VariantGear), Named("x", "l1"), BaseKey)), Is.True); - var refused = Refused(() => a.Commit(batch => + var refused = Assert.Throws(() => a.Commit(batch => { batch.Expect(other, a.Get(other)); batch.UpsertVariant(BigKey, BaseKey, new[] @@ -971,8 +1264,9 @@ namespace GameCult.Caching.Tests a.Override(nameof(VariantGear.Name), "x"), a.Override(nameof(VariantGear.Code), "big") }); - })); - Assert.That(refused.Message, Does.Contain(BigKey.Value).And.Contain(BaseKey.Value).And.Contain("x")); + }))!; + Assert.That(refused.Message, Does.Contain(BigKey.Value)); + Assert.That(refused.ChangedKeys, Is.EqualTo(new[] { BaseKey.Value }), "the refusal names the base another writer renamed"); } using var reopened = Open(path, readOnly: true); @@ -980,7 +1274,7 @@ namespace GameCult.Caching.Tests } [Test] - public void AConditionalCommitThatIsSoundOnTheMergedSetStillLands() + public void AConditionalCommitThatIsSoundOntoAVariantStoreAnotherWriterChangedIsRefusedThenLandsOnRetry() { var path = PathOf("gear.cc"); var other = new CultRecordKey("other"); @@ -991,11 +1285,15 @@ namespace GameCult.Caching.Tests using (var b = Open(path)) { SeedBig(a); - Assert.That(b.Commit(batch => + bool Write() => b.Commit(batch => { batch.Expect(other, null); batch.Upsert(typeof(VariantGear), Named("other", "fine"), other); - }), Is.True); + }); + + Assert.Throws(() => Write()); + Assert.That(b.Get(BigKey), Is.Not.Null, "the refusal reloaded the other writer's variant"); + Assert.That(Write(), Is.True); } using var reopened = Open(path); @@ -1111,7 +1409,7 @@ namespace GameCult.Caching.Tests } [Test] - public void AVariantAnotherWriterLandedIsStillJudgedWhenThisCacheHoldsNoVariant() + public void AVariantAnotherWriterLandedIsRefusedAsChangedWhenThisCacheHoldsNoVariant() { var path = PathOf("gear.cc"); var other = new CultRecordKey("other"); @@ -1122,7 +1420,7 @@ namespace GameCult.Caching.Tests using (var b = Open(path)) { VariantOnCode(a, "shared"); - Refused(() => b.Commit(batch => + Assert.Throws(() => b.Commit(batch => { batch.Expect(other, null); batch.Upsert(typeof(VariantGear), Named("other", "shared"), other); @@ -1374,7 +1672,7 @@ namespace GameCult.Caching.Tests public void AVariantRecordUnderAV1HeaderIsRefusedNamingTheRecord() { var path = WriteStore(CultPersistedStoreSnapshot.FormatV1, PlainGear("hand-base"), VariantGearRecord("hand-variant", "hand-base")); - var error = Assert.Throws(() => Open(path).Dispose())!; + var error = Assert.Throws(() => Open(path).Dispose())!; Assert.That(error.Message, Does.Contain("hand-variant").And.Contain("cultcache.store.v2")); } @@ -1382,7 +1680,7 @@ namespace GameCult.Caching.Tests public void AVariantRecordWithAPayloadIsRefusedNamingTheRecord() { var path = WriteStore(PlainGear("hand-base"), VariantGearRecord("hand-variant", "hand-base", payload: new byte[] { 0x90 })); - var error = Assert.Throws(() => Open(path).Dispose())!; + var error = Assert.Throws(() => Open(path).Dispose())!; Assert.That(error.Message, Does.Contain("hand-variant").And.Contain("payload")); } @@ -1390,7 +1688,7 @@ namespace GameCult.Caching.Tests public void AnOverrideWithAnUnknownOpIsRefusedNamingTheRecord() { var path = WriteStore(PlainGear("hand-base"), VariantGearRecord("hand-variant", "hand-base", op: (CultOverrideOp)7)); - var error = Assert.Throws(() => Open(path).Dispose())!; + var error = Assert.Throws(() => Open(path).Dispose())!; Assert.That(error.Message, Does.Contain("hand-variant").And.Contain("op 7")); } diff --git a/tests/GameCult.Caching.Tests/WriteSetTests.cs b/tests/GameCult.Caching.Tests/WriteSetTests.cs new file mode 100644 index 00000000..37217aef --- /dev/null +++ b/tests/GameCult.Caching.Tests/WriteSetTests.cs @@ -0,0 +1,1283 @@ +#nullable enable +using System; +using System.Buffers; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using GameCult.Caching.MessagePack; +using MessagePack; +using NUnit.Framework; + +namespace GameCult.Caching.Tests +{ + // Rule 1 of variants:ruling:write-set-content-version-declared-ownership: a flush, a commit and a single-document write apply + // only what the writer staged to what the store durably holds now, under its lock. Every other record and catalog entry is + // copied forward as stored. Every input goes through the cache's own write API and the real stores. + public class WriteSetTests + { + [CultDocument("tests.write_set_item", "tests.write_set_item.v1")] + [MessagePackObject(AllowPrivate = true)] + internal sealed class WsItem + { + [Key(0)] + [CultName] + public string Name = string.Empty; + + [Key(1)] + public string Note = string.Empty; + } + + [CultDocument("tests.write_set_other", "tests.write_set_other.v1")] + [MessagePackObject(AllowPrivate = true)] + internal sealed class WsOther + { + [Key(0)] + [CultName] + public string Name = string.Empty; + } + + [CultDocument("tests.write_set_nested", "tests.write_set_nested.v1")] + [MessagePackObject(AllowPrivate = true)] + internal sealed class WsNested + { + [Key(0)] + [CultName] + public string Name = string.Empty; + + [Key(1)] + public List Tags = new(); + + [Key(2)] + public WsInner Inner = new(); + } + + [MessagePackObject(AllowPrivate = true)] + internal sealed class WsInner + { + [Key(0)] + public int X; + } + + private static readonly CultDocumentRegistry Registry = CultDocumentRegistry.ForTypes(new[] { typeof(WsItem), typeof(WsOther), typeof(WsNested) }); + private static readonly string ItemId = Registry.GetRequired(typeof(WsItem)).SchemaId; + private static readonly CultRecordKey A = new("a"); + private static readonly CultRecordKey X = new("x"); + + private string _directory = string.Empty; + + [SetUp] + public void SetUp() + { + _directory = Path.Combine(Path.GetTempPath(), "cultlib-write-set-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(_directory); + } + + [TearDown] + public void TearDown() + { + if (Directory.Exists(_directory)) + Directory.Delete(_directory, true); + } + + private string PathOf(string name) => Path.Combine(_directory, name); + + private static CultCache Open(string path, bool directory) => + CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry, UseDirectoryStore = directory }); + + private static CultPersistedStoreSnapshot Read(string path) => + CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + + private static void Write(string path, CultPersistedStoreSnapshot snapshot) => + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + + // What a record is on disk: its id, its storedAt and, in a single file, its payload (a directory manifest indexes pages). + private static string Stored(string path, string key, bool directory) + { + var record = Read(path).Records.Single(candidate => candidate.Key == key); + return $"{record.SchemaId}|{record.StoredAt}|{(directory ? string.Empty : BitConverter.ToString(record.Payload))}"; + } + + private static byte[] EntryBytes(string path, string schemaId) => + StoreSlices.CatalogEntry(path, schemaId); + + private static string[] Keys(string path) => Read(path).Records.Select(record => record.Key).OrderBy(key => key, StringComparer.Ordinal).ToArray(); + + private static void Seed(string path, bool directory, params string[] keys) + { + using var cache = Open(path, directory); + cache.Commit(batch => + { + foreach (var key in keys) + batch.Upsert(typeof(WsItem), new WsItem { Name = "name-" + key, Note = "seeded" }, new CultRecordKey(key)); + }); + } + + // One staged write landed by a plain flush or by an unconditional commit. + private static void Land(CultCache cache, bool commit, Type type, object document, CultRecordKey key) + { + if (commit) + { + cache.Commit(batch => batch.Upsert(type, document, key)); + return; + } + + cache.UpsertAsync(type, document, key).GetAwaiter().GetResult(); + cache.FlushAllBackingStores(); + } + + // A loads, B writes x, A stages y and flushes or commits: x keeps B's bytes, id and storedAt. A record B removed stays removed. + [Test] + public void ALoadThenAWriteAfterAnotherWritersWriteKeepsThatWrite([Values] bool directory, [Values] bool commit) + { + var path = PathOf("keeps.cc"); + Seed(path, directory, "a"); + using var a = Open(path, directory); + using (var b = Open(path, directory)) + b.Commit(batch => batch.Upsert(typeof(WsItem), new WsItem { Name = "name-x", Note = "by-b" }, X)); + var written = Stored(path, "x", directory); + + Land(a, commit, typeof(WsItem), new WsItem { Name = "name-y", Note = "by-a" }, new CultRecordKey("y")); + + Assert.That(Keys(path), Is.EqualTo(new[] { "a", "x", "y" })); + Assert.That(Stored(path, "x", directory), Is.EqualTo(written)); + } + + [Test] + public void ARecordAnotherWriterRemovedStaysRemovedAfterALaterWrite([Values] bool directory, [Values] bool commit) + { + var path = PathOf("removed.cc"); + Seed(path, directory, "a", "x"); + using var a = Open(path, directory); + Assert.That(a.Get(X), Is.Not.Null, "A loaded x"); + using (var b = Open(path, directory)) + b.Commit(batch => batch.Remove(X)); + + Land(a, commit, typeof(WsItem), new WsItem { Name = "name-y" }, new CultRecordKey("y")); + + Assert.That(Keys(path), Is.EqualTo(new[] { "a", "y" }), "A's copy of x is not written back"); + } + + // A path with nothing at it when the cache opened, and a store there by the time it writes: the write lands on that store. + [Test] + public void AWriteOverAStoreThatAppearedAfterTheOpenKeepsItsRecords([Values] bool commit) + { + var path = PathOf("appeared.cc"); + using var a = Open(path, false); + Assert.That(File.Exists(path), Is.False); + using (var b = Open(path, false)) + b.Commit(batch => batch.Upsert(typeof(WsItem), new WsItem { Name = "name-x" }, X)); + + Land(a, commit, typeof(WsItem), new WsItem { Name = "name-y" }, new CultRecordKey("y")); + + Assert.That(Keys(path), Is.EqualTo(new[] { "x", "y" })); + } + + // An in-place change to a loaded document is not a write: it is written when the document is staged. + [Test] + public void AnInPlaceChangeThatWasNeverStagedIsNotWrittenAndAStagedOneIs([Values] bool directory, [Values] bool commit) + { + var path = PathOf("in-place.cc"); + Seed(path, directory, "a"); + using var cache = Open(path, directory); + var item = cache.Get(A)!; + item.Note = "edited"; + + cache.FlushAllBackingStores(); + using (var check = Open(path, directory)) + Assert.That(check.Get(A)!.Note, Is.EqualTo("seeded"), "an unstaged change is not written"); + + Land(cache, commit, typeof(WsItem), item, A); + using (var check = Open(path, directory)) + Assert.That(check.Get(A)!.Note, Is.EqualTo("edited")); + } + + // B changes the durable entry for the id of a record A carries. A writes a record under another id: the entry for the id it + // did not write is B's, byte for byte, not the one A registered or loaded. + [Test] + public void ARecordCopiedForwardKeepsTheCatalogEntryTheStoreHoldsNow([Values] bool directory, [Values] bool commit) + { + var path = PathOf("entry.cc"); + Seed(path, directory, "a"); + using var cache = Open(path, directory); + var snapshot = Read(path); + snapshot.SchemaCatalog.Single(entry => entry.SchemaId == ItemId).ContentHash = "changed-by-b"; + Write(path, snapshot); + var theirs = EntryBytes(path, ItemId); + + Land(cache, commit, typeof(WsOther), new WsOther { Name = "name-z" }, new CultRecordKey("z")); + + Assert.That(EntryBytes(path, ItemId), Is.EqualTo(theirs)); + Assert.That(Read(path).SchemaCatalog.Single(entry => entry.SchemaId == ItemId).ContentHash, Is.EqualTo("changed-by-b")); + } + + // The runtime decodes seven fields of an entry. Fields after them, and the width a value was written in, are not its to drop. + [Test] + public void ACarriedCatalogEntryKeepsItsExactBytesIncludingWhatThisRuntimeDoesNotDecode([Values] bool directory, [Values] bool commit) + { + var path = PathOf("raw.cc"); + Seed(path, directory, "a"); + var original = EntryBytes(path, ItemId); + Assert.That(original[0], Is.EqualTo(0x97), "a fixarray of the seven fields the runtime knows"); + var crafted = new byte[] { 0x99 } + .Concat(original.Skip(1)) + .Concat(new byte[] { 0xd3, 0, 0, 0, 0, 0, 0, 0, 1 }) + .Concat(new byte[] { 0xc0 }) + .ToArray(); + var bytes = File.ReadAllBytes(path); + var at = IndexOf(bytes, original); + Assert.That(at, Is.GreaterThanOrEqualTo(0)); + File.WriteAllBytes(path, bytes.Take(at).Concat(crafted).Concat(bytes.Skip(at + original.Length)).ToArray()); + + using var cache = Open(path, directory); + Assert.That(cache.Get(A), Is.Not.Null, "the entry with two unknown trailing fields reads"); + Land(cache, commit, typeof(WsOther), new WsOther { Name = "name-z" }, new CultRecordKey("z")); + + Assert.That(IndexOf(File.ReadAllBytes(path), crafted), Is.GreaterThanOrEqualTo(0), "the entry is in the written store byte for byte"); + } + + private static int IndexOf(byte[] haystack, byte[] needle) + { + for (var start = 0; start + needle.Length <= haystack.Length; start++) + { + var same = true; + for (var index = 0; same && index < needle.Length; index++) + same = haystack[start + index] == needle[index]; + if (same) + return start; + } + + return -1; + } + + // The condition is the only difference between a conditional and an unconditional commit: one batch, one apply. + [Test] + public void AConditionalAndAnUnconditionalCommitOfOneBatchOntoOneStoreWriteTheSameBytes() + { + var first = PathOf("unconditional.cc"); + var second = PathOf("conditional.cc"); + Seed(first, false, "a", "x"); + File.Copy(first, second); + + foreach (var (path, conditional) in new[] { (first, false), (second, true) }) + { + using var cache = Open(path, false); + var current = cache.Get(A)!; + Assert.That(cache.Commit(batch => + { + if (conditional) + batch.Expect(A, current); + batch.Upsert(typeof(WsOther), new WsOther { Name = "name-z" }, new CultRecordKey("z")); + batch.Remove(X); + }), Is.True); + } + + byte[] Normalized(string path) + { + var snapshot = Read(path); + foreach (var record in snapshot.Records.Where(record => record.Key == "z")) + record.StoredAt = string.Empty; + return CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot); + } + + Assert.That(Keys(first), Is.EqualTo(new[] { "a", "z" })); + Assert.That(Normalized(second), Is.EqualTo(Normalized(first))); + } + + // An unconditional commit also lands what was staged before it, on the store as it is now. + [Test] + public void AnUnconditionalCommitLandsEarlierStagedWritesWithItsBatchOntoWhatAnotherWriterWrote([Values] bool directory) + { + var path = PathOf("staged.cc"); + Seed(path, directory, "a"); + using var cache = Open(path, directory); + using (var other = Open(path, directory)) + other.Commit(batch => batch.Upsert(typeof(WsItem), new WsItem { Name = "name-x" }, X)); + cache.UpsertAsync(typeof(WsItem), new WsItem { Name = "name-s" }, new CultRecordKey("s")).GetAwaiter().GetResult(); + + Assert.That(cache.Commit(batch => batch.Upsert(typeof(WsItem), new WsItem { Name = "name-y" }, new CultRecordKey("y"))), Is.True); + + Assert.That(Keys(path), Is.EqualTo(new[] { "a", "s", "x", "y" })); + Assert.That(cache.IsDirty, Is.False); + } + + // The operator's check on a real store: CULTLIB_REAL_STORE names a copy of one. A cache that registers none of its types + // stages one record and flushes or commits: every other record and catalog entry, and the header, are byte for byte as they were. + [Test] + public void ARealStoreSurvivesOneStagedWriteByteForByte([Values] bool commit) + { + var source = Environment.GetEnvironmentVariable("CULTLIB_REAL_STORE"); + if (string.IsNullOrEmpty(source) || !File.Exists(source)) + Assert.Ignore("CULTLIB_REAL_STORE does not name a copy of a real store."); + var path = PathOf("real.cc"); + File.Copy(source!, path); + var beforePath = PathOf("real-before.cc"); + File.Copy(source!, beforePath); + var before = Read(path); + + using (var cache = Open(path, false)) + Land(cache, commit, typeof(WsItem), new WsItem { Name = "name-real-write" }, new CultRecordKey("real-write")); + + var after = Read(path); + TestContext.Out.WriteLine($"real store: {before.Records.Length} records, {before.SchemaCatalog.Length} catalog entries, header {before.FormatVersion}"); + Assert.That(after.Records.Length, Is.EqualTo(before.Records.Length + 1)); + Assert.That(after.FormatVersion, Is.EqualTo(before.FormatVersion)); + + var changed = before.Records.Where(record => !StoreSlices.Record(beforePath, record.Key).SequenceEqual(StoreSlices.Record(path, record.Key))).Select(record => record.Key).ToArray(); + Assert.That(changed, Is.Empty, "records that are not byte-identical"); + var afterEntries = StoreSlices.CatalogEntries(path).ToList(); + var missing = StoreSlices.CatalogEntries(beforePath).Where(raw => !afterEntries.Any(other => other.SequenceEqual(raw))).Select(raw => BitConverter.ToString(raw)).ToArray(); + Assert.That(missing, Is.Empty, "catalog entries that are not byte-identical"); + } + + private static void MarkAs(string path, string header) + { + var snapshot = Read(path); + snapshot.FormatVersion = header; + Write(path, snapshot); + } + + // A marked header says the store holds an element id, and a record's bytes cannot say whether it does. A write that copies + // any record forward therefore keeps the mark, whatever this cache wrote or read of that record. + [Test] + public void AMarkedHeaderIsKeptWhileTheWriteCopiesAnyRecord([Values] bool anotherWriterAddedARecord) + { + var path = PathOf("mark.cc"); + Seed(path, false, "a"); + MarkAs(path, CultPersistedStoreSnapshot.FormatV3); + using var cache = Open(path, false); + if (anotherWriterAddedARecord) + { + using (var other = Open(path, false)) + other.Commit(batch => batch.Upsert(typeof(WsItem), new WsItem { Name = "name-x" }, X)); + MarkAs(path, CultPersistedStoreSnapshot.FormatV3); + } + + Land(cache, commit: false, typeof(WsItem), new WsItem { Name = "name-y" }, new CultRecordKey("y")); + + Assert.That(Read(path).FormatVersion, Is.EqualTo(CultPersistedStoreSnapshot.FormatV3)); + } + + // Only a write that encodes every record the store will hold can say none holds an id. + [Test] + public void AMarkedHeaderIsDroppedOnlyByAWriteThatEncodesEveryRecord([Values] bool commit) + { + var path = PathOf("mark-all.cc"); + Seed(path, false, "a"); + MarkAs(path, CultPersistedStoreSnapshot.FormatV3); + using var cache = Open(path, false); + + Land(cache, commit, typeof(WsItem), new WsItem { Name = "name-a", Note = "again" }, A); + + Assert.That(Read(path).FormatVersion, Is.EqualTo(CultPersistedStoreSnapshot.FormatV1)); + } + + // Another writer changed a copied record after this cache read it, and the mark stays: a copied record is not read to decide it. + [Test] + public void AMarkedHeaderIsKeptWhenAnotherWriterChangedACopiedRecordAfterTheLoad() + { + var path = PathOf("mark-changed.cc"); + Seed(path, false, "a"); + MarkAs(path, CultPersistedStoreSnapshot.FormatV3); + using var cache = Open(path, false); + using (var other = Open(path, false)) + other.Commit(batch => batch.Upsert(typeof(WsItem), new WsItem { Name = "name-a", Note = "changed" }, A)); + MarkAs(path, CultPersistedStoreSnapshot.FormatV3); + + Land(cache, commit: false, typeof(WsItem), new WsItem { Name = "name-y" }, new CultRecordKey("y")); + + Assert.That(Read(path).FormatVersion, Is.EqualTo(CultPersistedStoreSnapshot.FormatV3)); + } + + // A copied record stays copied whoever wrote it: what this cache wrote earlier does not make the next write one that decides by content. + [Test] + public void AMarkedHeaderIsKeptWhenEveryCopiedRecordWasWrittenByThisCache() + { + var path = PathOf("mark-wrote.cc"); + Seed(path, false, "a"); + using var cache = Open(path, false); + Land(cache, commit: false, typeof(WsItem), new WsItem { Name = "name-y" }, new CultRecordKey("y")); + MarkAs(path, CultPersistedStoreSnapshot.FormatV3); + + Land(cache, commit: false, typeof(WsItem), new WsItem { Name = "name-z" }, new CultRecordKey("z")); + + Assert.That(Read(path).FormatVersion, Is.EqualTo(CultPersistedStoreSnapshot.FormatV3)); + } + + // A batch is the write that lands last: it replaces a staged write of the same key. + [Test] + public void ABatchWinsOverAStagedWriteOfTheSameKey() + { + var path = PathOf("batch-wins.cc"); + Seed(path, false, "a"); + using (var cache = Open(path, false)) + { + cache.UpsertAsync(typeof(WsItem), new WsItem { Name = "name-a", Note = "staged" }, A).GetAwaiter().GetResult(); + Assert.That(cache.Commit(batch => batch.Upsert(typeof(WsItem), new WsItem { Name = "name-a", Note = "batch" }, A)), Is.True); + } + + using var check = Open(path, false); + Assert.That(check.Get(A)!.Note, Is.EqualTo("batch")); + } + + // v2 exactly when the written store holds a variant: a copied variant keeps it, and the last one removed drops it. + [Test] + public void TheV2HeaderIsWrittenExactlyWhileTheStoreHoldsAVariant() + { + var path = PathOf("variant.cc"); + Seed(path, false, "a"); + var variant = new CultRecordKey("v"); + using (var cache = Open(path, false)) + { + cache.Commit(batch => batch.UpsertVariant(variant, A, new[] + { + cache.Override(nameof(WsItem.Name), "name-v"), + cache.Override(nameof(WsItem.Note), "variant") + })); + } + + Assert.That(Read(path).FormatVersion, Is.EqualTo(CultPersistedStoreSnapshot.FormatV2)); + + using (var cache = Open(path, false)) + { + Land(cache, commit: true, typeof(WsItem), new WsItem { Name = "name-y" }, new CultRecordKey("y")); + Assert.That(Read(path).FormatVersion, Is.EqualTo(CultPersistedStoreSnapshot.FormatV2), "the variant was copied"); + cache.Commit(batch => batch.Remove(variant)); + } + + Assert.That(Read(path).FormatVersion, Is.EqualTo(CultPersistedStoreSnapshot.FormatV1)); + } + + // Another writer adds a record under this cache's schema id whose payload this cache's type cannot decode. + private static void AddUndecodableRecord(string path, string key) + { + var snapshot = Read(path); + snapshot.Records = snapshot.Records.Concat(new[] + { + new CultPersistedRecord { Key = key, SchemaId = ItemId, StoredAt = DateTimeOffset.UtcNow.ToString("O"), Payload = new byte[] { 0xC1 } } + }).OrderBy(record => record.Key, StringComparer.Ordinal).ToArray(); + Write(path, snapshot); + } + + // A flush decodes nothing it did not stage: a copied record this cache's type cannot decode is carried byte for byte. + [Test] + public void AFlushCopiesARecordThisCacheCannotDecode([Values] bool commit) + { + var path = PathOf("undecodable.cc"); + Seed(path, false, "a"); + using var a = Open(path, false); + AddUndecodableRecord(path, "r"); + var carried = Stored(path, "r", false); + + Land(a, commit, typeof(WsItem), new WsItem { Name = "name-y" }, new CultRecordKey("y")); + + Assert.That(Keys(path), Is.EqualTo(new[] { "a", "r", "y" })); + Assert.That(Stored(path, "r", false), Is.EqualTo(carried)); + } + + // The same record in a store holding a variant cannot be judged without decoding it: the write is refused with the decode + // failure as its cause, and the disk, the cache and its staged change are as they were. + [Test] + public void AWriteIntoAVariantStoreWithARecordThisCacheCannotLoadChangesNothing([Values] bool commit) + { + var path = PathOf("undecodable-variant.cc"); + Seed(path, false, "a"); + using (var seed = Open(path, false)) + seed.Commit(batch => batch.UpsertVariant(new CultRecordKey("v"), A, new[] { seed.Override(nameof(WsItem.Name), "name-v"), seed.Override(nameof(WsItem.Note), "variant") })); + var a = Open(path, false); + AddUndecodableRecord(path, "r"); + var before = File.ReadAllBytes(path); + var y = new CultRecordKey("y"); + + var conflict = Assert.Throws(() => + { + if (commit) + a.Commit(batch => batch.Upsert(typeof(WsItem), new WsItem { Name = "name-y" }, y)); + else + Land(a, false, typeof(WsItem), new WsItem { Name = "name-y" }, y); + })!; + + Assert.That(conflict.InnerException, Is.Not.Null, "the decode failure is the cause"); + Assert.That(conflict.ChangedKeys, Is.EqualTo(new[] { "r" })); + Assert.That(File.ReadAllBytes(path), Is.EqualTo(before)); + Assert.That(a.Get(new CultRecordKey("r")), Is.Null); + if (!commit) + { + Assert.That(a.IsDirty, Is.True, "a reload that fails changes nothing, the staged write included"); + Assert.That(a.Get(y), Is.Not.Null); + } + } + + // A stages k, B writes k later, A flushes: A's bytes land at a storedAt later than B's in either store kind, so a writer holding B's k is told it moved. + [Test] + public void AWriteOverAnotherWritersLaterRecordStoresALaterStoredAt([Values] bool directory) + { + var path = PathOf("later.cc"); + Seed(path, directory, "a"); + var k = new CultRecordKey("k"); + using var a = Open(path, directory); + a.UpsertAsync(typeof(WsItem), new WsItem { Name = "name-k", Note = "by-a" }, k).GetAwaiter().GetResult(); + System.Threading.Thread.Sleep(30); + using var b = Open(path, directory); + b.Commit(batch => batch.Upsert(typeof(WsItem), new WsItem { Name = "name-k", Note = "by-b" }, k)); + var seenByB = b.Get(k)!; + var bAt = Read(path).Records.Single(record => record.Key == "k").StoredAt; + + a.FlushAllBackingStores(); + + var onDisk = Read(path).Records.Single(record => record.Key == "k"); + Assert.That(DateTimeOffset.Parse(onDisk.StoredAt, null, System.Globalization.DateTimeStyles.RoundtripKind), + Is.GreaterThan(DateTimeOffset.Parse(bAt, null, System.Globalization.DateTimeStyles.RoundtripKind))); + Assert.That(a.Commit(batch => { batch.Expect(k, a.Get(k)!); batch.Upsert(typeof(WsItem), new WsItem { Name = "name-k", Note = "again" }, k); }), + Is.True, "A's own condition holds without a pull"); + Assert.That(b.Commit(batch => { batch.Expect(k, seenByB); batch.Upsert(typeof(WsItem), new WsItem { Name = "name-k", Note = "stale" }, k); }), + Is.False, "B's copy of k is no longer what the store holds"); + } + + // Rule 2: a record's version is the SHA-256 of its stored bytes. Another writer rewrites the record to other bytes yet leaves + // its storedAt as it was (a peer whose clock equals, or a tool): the bytes are all that tells the two records apart. + private void RewriteAtItsOwnStoredAt(string path, bool directory, string key, string note) + { + var storedAt = StoredAtOf(path, key); + using (var writer = Open(path, directory)) + writer.Commit(batch => batch.Upsert(typeof(WsItem), new WsItem { Name = "name-" + key, Note = note }, new CultRecordKey(key))); + var snapshot = Read(path); + var record = snapshot.Records.Single(candidate => candidate.Key == key); + if (!directory) + { + record.StoredAt = storedAt; + Write(path, snapshot); + return; + } + + var pages = DirectoryMessagePackBackingStore.DefaultRecordDirectoryPath(path); + var page = CultDocumentMessagePackSerialization.DeserializePersistedRecord( + File.ReadAllBytes(Path.Combine(pages, Convert.ToHexString(record.Payload).ToLowerInvariant() + ".msgpack"))); + page.StoredAt = storedAt; + var bytes = CultDocumentMessagePackSerialization.SerializePersistedRecord(page); + var hash = System.Security.Cryptography.SHA256.HashData(bytes); + File.WriteAllBytes(Path.Combine(pages, Convert.ToHexString(hash).ToLowerInvariant() + ".msgpack"), bytes); + record.StoredAt = storedAt; + record.Payload = hash; + Write(path, snapshot); + } + + private static string NoteOnDisk(string path, bool directory) + { + using var fresh = Open(path, directory); + return fresh.Get(K)!.Note; + } + + private static readonly CultRecordKey K = new("k"); + + // A reads k; another writer rewrites it at its own storedAt; A's Expect on what it read fails and the disk keeps the rewrite. + [Test] + public void AStaleExpectFailsWhenAnotherWriterRewroteTheRecordAtItsOwnStoredAt([Values] bool directory) + { + var path = PathOf("t1.cc"); + Seed(path, directory, "k"); + using var a = Open(path, directory); + var observed = a.Get(K)!; + RewriteAtItsOwnStoredAt(path, directory, "k", "by-c"); + + var committed = a.Commit(batch => + { + batch.Expect(K, observed); + batch.Upsert(typeof(WsItem), new WsItem { Name = "name-k", Note = "stale" }, K); + }); + + Assert.That(committed, Is.False); + Assert.That(NoteOnDisk(path, directory), Is.EqualTo("by-c")); + } + + // A pull serves the rewritten record although nothing about it but its bytes changed. + [Test] + public void APullServesARecordRewrittenAtItsOwnStoredAt([Values] bool directory) + { + var path = PathOf("t2.cc"); + Seed(path, directory, "k"); + using var a = Open(path, directory); + RewriteAtItsOwnStoredAt(path, directory, "k", "by-c"); + + a.PullAllBackingStoresAsync().GetAwaiter().GetResult(); + + Assert.That(a.Get(K)!.Note, Is.EqualTo("by-c")); + } + + // ExpectUnchanged compares every record's version: a rewrite at the same storedAt fails it for an unrelated key. + [Test] + public void ExpectUnchangedFailsWhenARecordWasRewrittenAtItsOwnStoredAt([Values] bool directory) + { + var path = PathOf("t3.cc"); + Seed(path, directory, "k"); + using var a = Open(path, directory); + RewriteAtItsOwnStoredAt(path, directory, "k", "by-c"); + + var committed = a.Commit(batch => + { + batch.ExpectUnchanged(); + batch.Upsert(typeof(WsItem), new WsItem { Name = "name-x" }, X); + }); + + Assert.That(committed, Is.False); + Assert.That(Keys(path), Is.EqualTo(new[] { "k" })); + } + + // The staleness check of a store holding a variant compares versions: a rewrite at the record's own storedAt refuses the + // write, names the record, leaves the file as it was, and the cache then holds the rewrite so a retry lands. + [Test] + public void AWriteIntoAVariantStoreIsRefusedWhenARecordWasRewrittenAtItsOwnStoredAt() + { + var path = PathOf("t4.cc"); + Seed(path, false, "a"); + using var a = Open(path, false); + AddVariant(a, "v", A); + using (var peer = Open(path, false)) + Assert.That(peer.Get(A), Is.Not.Null); + RewriteAtItsOwnStoredAt(path, false, "a", "by-c"); + var before = File.ReadAllBytes(path); + var y = new CultRecordKey("y"); + + var conflict = Assert.Throws(() => + Land(a, true, typeof(WsItem), new WsItem { Name = "name-y" }, y))!; + + Assert.That(conflict.ChangedKeys, Is.EqualTo(new[] { "a" })); + Assert.That(File.ReadAllBytes(path), Is.EqualTo(before)); + Assert.That(a.Get(A)!.Note, Is.EqualTo("by-c"), "the refusal reloaded the record by version"); + Land(a, true, typeof(WsItem), new WsItem { Name = "name-y" }, y); + Assert.That(Keys(path), Is.EqualTo(new[] { "a", "v", "y" })); + } + + // A write teaches the writer's cache the version of the bytes it wrote, by flush or by commit, in either store kind: the + // writer's own Expect holds without a pull, and fails once anyone else writes the record. + [Test] + public void AnOwnWriteTeachesTheCacheItsVersion([Values] bool directory, [Values] bool commit) + { + var path = PathOf("t5.cc"); + Seed(path, directory, "a"); + using var a = Open(path, directory); + Land(a, commit, typeof(WsItem), new WsItem { Name = "name-k", Note = "by-a" }, K); + + Assert.That(a.Commit(batch => + { + batch.Expect(K, a.Get(K)!); + batch.Upsert(typeof(WsItem), new WsItem { Name = "name-k", Note = "again" }, K); + }), Is.True, "A's own condition holds without a pull"); + + var seen = a.Get(K)!; + using (var b = Open(path, directory)) + b.Commit(batch => batch.Upsert(typeof(WsItem), new WsItem { Name = "name-k", Note = "by-b" }, K)); + Assert.That(a.Commit(batch => + { + batch.Expect(K, seen); + batch.Upsert(typeof(WsItem), new WsItem { Name = "name-k", Note = "stale" }, K); + }), Is.False); + } + + // A version is the SHA-256 of the bytes as the file holds them, not of a re-encoding. A writer may legally encode a record + // wider than needed (a str16 for a short string, a bin32, an array16); the file below holds every record that way. Each + // held record's version is the hash of its slice of the file; a pull over the unchanged file replaces nothing; and a write + // copies the records it did not stage byte for byte, so they keep their version and the next write is not refused. + [Test] + public void AVersionIsTheHashOfTheBytesTheFileHoldsWhateverTheirEncoding() + { + var path = PathOf("wide.cc"); + Seed(path, false, "a", "k"); + using (var seed = Open(path, false)) + AddVariant(seed, "v", A); + File.WriteAllBytes(path, WidenEveryRecord(File.ReadAllBytes(path), out var slices)); + var wide = Read(path).Records.Single(record => record.Key == "k"); + Assert.That(slices["k"], Is.Not.EqualTo(CultDocumentMessagePackSerialization.SerializePersistedRecord(new CultPersistedRecord + { + Key = wide.Key, + SchemaId = wide.SchemaId, + StoredAt = wide.StoredAt, + Payload = wide.Payload + })), "the stored bytes are not the canonical encoding of the record"); + + using var cache = Open(path, false); + foreach (var (key, slice) in slices) + Assert.That(cache.GetStored(new CultRecordKey(key))!.StoredVersion, Is.EqualTo(HexOfSha256(slice)), key); + + var before = cache.GetStored(K); + cache.PullAllBackingStoresAsync().GetAwaiter().GetResult(); + cache.PullAllBackingStoresAsync().GetAwaiter().GetResult(); + Assert.That(ReferenceEquals(cache.GetStored(K), before), Is.True, "an unchanged file is not read as changed"); + + Assert.That(cache.Commit(batch => + { + batch.Expect(K, cache.Get(K)!); + batch.Upsert(typeof(WsItem), new WsItem { Name = "name-x" }, X); + }), Is.True); + var file = File.ReadAllBytes(path); + foreach (var key in new[] { "a", "v" }) + Assert.That(Contains(file, slices[key]), Is.True, $"{key} was copied as the file held it"); + Land(cache, false, typeof(WsItem), new WsItem { Name = "name-x", Note = "again" }, X); + Assert.That(Keys(path), Is.EqualTo(new[] { "a", "k", "v", "x" }), "a second write into the variant store is not refused"); + } + + private static string HexOfSha256(byte[] bytes) => + Convert.ToHexString(System.Security.Cryptography.SHA256.HashData(bytes)).ToLowerInvariant(); + + private static bool Contains(byte[] haystack, byte[] needle) => IndexOf(haystack, needle) >= 0; + + // The same store with every array, string and byte string written in its widest MessagePack form, and each record's slice. + private static byte[] WidenEveryRecord(byte[] bytes, out Dictionary slices) + { + slices = new Dictionary(StringComparer.Ordinal); + var output = new List(); + var reader = new MessagePackReader(bytes); + var top = reader.ReadArrayHeader(); + output.AddRange(new byte[] { 0xdc, (byte)(top >> 8), (byte)top }); + for (var slot = 0; slot < top; slot++) + { + if (slot != 2) + { + var from = (int)reader.Consumed; + reader.Skip(); + output.AddRange(bytes.Skip(from).Take((int)reader.Consumed - from)); + continue; + } + + var count = reader.ReadArrayHeader(); + output.AddRange(new byte[] { 0xdc, (byte)(count >> 8), (byte)count }); + for (var index = 0; index < count; index++) + { + var start = output.Count; + var fields = reader.ReadArrayHeader(); + output.AddRange(new byte[] { 0xdc, (byte)(fields >> 8), (byte)fields }); + string? key = null; + for (var field = 0; field < fields; field++) + { + if (reader.NextMessagePackType == MessagePackType.String) + { + var text = reader.ReadString()!; + key ??= text; + var utf8 = System.Text.Encoding.UTF8.GetBytes(text); + output.AddRange(new byte[] { 0xda, (byte)(utf8.Length >> 8), (byte)utf8.Length }); + output.AddRange(utf8); + } + else if (reader.NextMessagePackType == MessagePackType.Binary) + { + var blob = reader.ReadBytes()?.ToArray() ?? Array.Empty(); + output.AddRange(new byte[] { 0xc6, (byte)(blob.Length >> 24), (byte)(blob.Length >> 16), (byte)(blob.Length >> 8), (byte)blob.Length }); + output.AddRange(blob); + } + else + { + var from = (int)reader.Consumed; + reader.Skip(); + output.AddRange(bytes.Skip(from).Take((int)reader.Consumed - from)); + } + } + + slices[key!] = output.Skip(start).ToArray(); + } + } + + return output.ToArray(); + } + + // A condition on a record fails when another writer removed it: the removal stands and the writer does not bring it back. + [Test] + public void AnExpectFailsWhenAnotherWriterRemovedTheRecord([Values] bool directory) + { + var path = PathOf("gone.cc"); + Seed(path, directory, "k"); + using var a = Open(path, directory); + var observed = a.Get(K)!; + using (var b = Open(path, directory)) + Assert.That(b.Commit(batch => batch.Remove(K)), Is.True); + + var committed = a.Commit(batch => + { + batch.Expect(K, observed); + batch.Upsert(typeof(WsItem), new WsItem { Name = "name-k", Note = "resurrected" }, K); + }); + + Assert.That(committed, Is.False); + Assert.That(Keys(path), Is.Empty); + } + + // The cache's own held record, not only the store's, learns the version of the bytes written. A variant staged and then + // re-resolved (its base edited before the flush) is held by the cache as a new object the store never saw, so only the + // cache's own line gives it the version of the bytes it was written as. + [Test] + public void AVariantReResolvedBeforeItsFlushIsKnownByItsWrittenVersion() + { + var path = PathOf("reresolved.cc"); + Seed(path, false, "a"); + using var cache = Open(path, false); + var v = new CultRecordKey("v"); + cache.UpsertVariantAsync(v, A, new[] { cache.Override(nameof(WsItem.Name), "name-v") }).GetAwaiter().GetResult(); + var staged = cache.GetStored(v); + cache.UpsertAsync(typeof(WsItem), new WsItem { Name = "name-a", Note = "edited" }, A).GetAwaiter().GetResult(); + Assert.That(ReferenceEquals(cache.GetStored(v), staged), Is.False, "the base edit re-resolved the variant into a new held copy"); + cache.FlushAllBackingStores(); + + var held = cache.GetStored(v)!; + Assert.That(held.StoredVersion, Is.EqualTo(HexOfSha256(CultDocumentMessagePackSerialization.SerializePersistedRecord( + Read(path).Records.Single(record => record.Key == "v"))))); + Assert.That(cache.Commit(batch => + { + batch.Expect(v, cache.Get(v)!); + batch.Upsert(typeof(WsItem), new WsItem { Name = "name-x" }, X); + }), Is.True, "the writer's own condition on the variant holds"); + } + + + // A commit with no condition decides nothing about the other records, so it hashes none of them: only the record it writes + // is hashed to be named. A commit with a condition compares every durable record's version. + [Test] + public void AnUnconditionalCommitHashesOnlyTheRecordsItWrites() + { + var path = PathOf("cost.cc"); + Seed(path, false, "a", "b", "c", "d", "e"); + var store = new SingleFileMessagePackBackingStore(path); + using var cache = new CultCache(Registry, CultCacheMessagePack.CreateCodec(Registry)); + cache.AddBackingStore(store); + cache.PullAllBackingStoresAsync().GetAwaiter().GetResult(); + + store.RecordsHashed = 0; + Assert.That(cache.Commit(batch => batch.Upsert(typeof(WsItem), new WsItem { Name = "name-x" }, X)), Is.True); + Assert.That(store.RecordsHashed, Is.EqualTo(1)); + + store.RecordsHashed = 0; + Assert.That(cache.Commit(batch => + { + batch.Expect(X, cache.Get(X)!); + batch.Upsert(typeof(WsItem), new WsItem { Name = "name-x", Note = "again" }, X); + }), Is.True); + Assert.That(store.RecordsHashed, Is.GreaterThanOrEqualTo(6), "a condition hashes every durable record, which is how the count proves itself"); + } + + // Records no store has read or written are told apart however many are made in one clock tick. + [Test] + public void UnstoredRecordsMadeBackToBackHaveDistinctVersions() + { + using var cache = new CultCache(Registry); + var keys = Enumerable.Range(0, 2000).Select(index => new CultRecordKey("r" + index)).ToArray(); + foreach (var key in keys) + cache.UpsertAsync(typeof(WsItem), new WsItem { Name = "same" }, key).GetAwaiter().GetResult(); + + Assert.That(keys.Select(key => cache.GetStored(key)!.StoredVersion).Distinct().Count(), Is.EqualTo(keys.Length)); + } + + // A write into a store holding a variant is refused when any record it read has moved, including one the writer is itself + // writing: it must not overwrite another writer's edit with a copy it read before that edit. + [Test] + public void AWriteIntoAVariantStoreIsRefusedWhenARecordItWritesMovedAfterItRead() + { + var path = PathOf("own-upsert.cc"); + Seed(path, false, "a"); + using var a = Open(path, false); + AddVariant(a, "v", A); + using (var peer = Open(path, false)) + Assert.That(peer.Commit(batch => batch.Upsert(typeof(WsItem), new WsItem { Name = "name-a", Note = "by-peer" }, A)), Is.True); + var before = File.ReadAllBytes(path); + + var conflict = Assert.Throws(() => + Land(a, true, typeof(WsItem), new WsItem { Name = "name-a", Note = "stale" }, A))!; + + Assert.That(conflict.ChangedKeys, Is.EqualTo(new[] { "a" })); + Assert.That(File.ReadAllBytes(path), Is.EqualTo(before)); + } + + // A commit that removes a key this cache staged removes it everywhere: the staged write does not survive the batch. + [Test] + public void ACommitThatRemovesAStagedKeyRemovesIt() + { + var path = PathOf("staged-removed.cc"); + Seed(path, false, "a"); + var k = new CultRecordKey("k"); + using (var cache = Open(path, false)) + { + cache.UpsertAsync(typeof(WsItem), new WsItem { Name = "name-k" }, k).GetAwaiter().GetResult(); + Assert.That(cache.Commit(batch => batch.Remove(k)), Is.True); + Assert.That(Keys(path), Is.EqualTo(new[] { "a" })); + Assert.That(cache.Get(k), Is.Null); + } + + using var reopened = Open(path, false); + Assert.That(reopened.Get(k), Is.Null); + } + + private static DateTimeOffset At(string storedAt) => DateTimeOffset.Parse(storedAt, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind); + + private static string StoredAtOf(string path, string key) => Read(path).Records.Single(record => record.Key == key).StoredAt; + + // The stored bytes belong to the store that read them, never to a record object: a record read through the public snapshot + // API is only a decoded copy, so an edit made to its payload in place is what a write of it stores. + [Test] + public void ARecordReadThroughTheSnapshotApiAndEditedInPlaceIsWrittenAsEdited() + { + var path = PathOf("edit-payload.cc"); + Seed(path, false, "k"); + var snapshot = Read(path); + OverwriteInPlace(snapshot.Records.Single().Payload, "seeded", "SEEDED"); + + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + + Assert.That(NoteOnDisk(path, false), Is.EqualTo("SEEDED")); + var record = CultDocumentMessagePackSerialization.DeserializePersistedRecord(StoreSlices.Record(path, "k")); + OverwriteInPlace(record.Payload, "SEEDED", "seeded"); + Assert.That( + CultDocumentMessagePackSerialization.DeserializePersistedRecord(CultDocumentMessagePackSerialization.SerializePersistedRecord(record)).Payload, + Is.EqualTo(record.Payload), "a single record is re-encoded from its fields too"); + } + + [Test] + public void AnOverrideReadThroughTheSnapshotApiAndEditedInPlaceIsWrittenAsEdited() + { + var path = PathOf("edit-override.cc"); + Seed(path, false, "a"); + using (var seed = Open(path, false)) + AddVariant(seed, "v", A); + var snapshot = Read(path); + var variant = snapshot.Records.Single(record => record.Key == "v").Variant!; + OverwriteInPlace(variant.Overrides.Single(entry => entry.Path[0].Slot == 1).Value, "variant-v", "VARIANT-v"); + + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + + using var fresh = Open(path, false); + Assert.That(fresh.Get(new CultRecordKey("v"))!.Note, Is.EqualTo("VARIANT-v")); + } + + // A held document edited in place (a list, a nested object) and put back as the same instance is written as edited, and a + // later write of a neighbour copies it as the file then holds it. + [Test] + public void AHeldDocumentEditedInPlaceAndPutBackIsWrittenAsEdited([Values] bool commit) + { + var path = PathOf("edit-held.cc"); + var n = new CultRecordKey("n"); + using (var seed = Open(path, false)) + seed.Commit(batch => batch.Upsert(typeof(WsNested), new WsNested { Name = "name-n", Tags = { "t1" }, Inner = { X = 1 } }, n)); + + using (var cache = Open(path, false)) + { + var held = cache.Get(n)!; + held.Tags.Add("t2"); + held.Inner.X = 9; + Land(cache, commit, typeof(WsNested), held, n); + } + + using (var fresh = Open(path, false)) + { + var read = fresh.Get(n)!; + Assert.That(read.Tags, Is.EqualTo(new[] { "t1", "t2" })); + Assert.That(read.Inner.X, Is.EqualTo(9)); + } + + var written = StoreSlices.Record(path, "n"); + using (var other = Open(path, false)) + Land(other, commit, typeof(WsItem), new WsItem { Name = "name-x" }, X); + Assert.That(StoreSlices.Record(path, "n"), Is.EqualTo(written), "the edit is copied as the file holds it"); + } + + // A record the writer did not stage keeps the bytes and the version its file holds, whatever encoding its writer chose, + // across any number of writes of its neighbours. + [Test] + public void AnUntouchedNonCanonicalRecordKeepsItsBytesAndVersionAcrossAWriteOfANeighbour([Values] bool commit) + { + var path = PathOf("untouched-wide.cc"); + Seed(path, false, "k"); + File.WriteAllBytes(path, WidenEveryRecord(File.ReadAllBytes(path), out var slices)); + using var cache = Open(path, false); + var version = cache.GetStored(K)!.StoredVersion; + Assert.That(version, Is.EqualTo(HexOfSha256(slices["k"]))); + + Land(cache, commit, typeof(WsItem), new WsItem { Name = "name-x" }, X); + Land(cache, commit, typeof(WsItem), new WsItem { Name = "name-y" }, new CultRecordKey("y")); + + Assert.That(StoreSlices.Record(path, "k"), Is.EqualTo(slices["k"])); + Assert.That(cache.GetStored(K)!.StoredVersion, Is.EqualTo(version)); + using var fresh = Open(path, false); + Assert.That(fresh.GetStored(K)!.StoredVersion, Is.EqualTo(version)); + } + + // A record the writer stages is encoded from its object, so a non-canonical record put back unchanged is written canonically + // and, being other bytes, has another version. + [Test] + public void AStagedNonCanonicalRecordPutBackUnchangedIsReEncodedAndGetsANewVersion([Values] bool commit) + { + var path = PathOf("staged-wide.cc"); + Seed(path, false, "k"); + File.WriteAllBytes(path, WidenEveryRecord(File.ReadAllBytes(path), out var slices)); + using var cache = Open(path, false); + var held = cache.Get(K)!; + + Land(cache, commit, typeof(WsItem), held, K); + + var written = StoreSlices.Record(path, "k"); + Assert.That(written, Is.Not.EqualTo(slices["k"])); + Assert.That(written, Is.EqualTo(CultDocumentMessagePackSerialization.SerializePersistedRecord(Read(path).Records.Single())), + "the record is in the canonical encoding"); + Assert.That(cache.GetStored(K)!.StoredVersion, Is.EqualTo(HexOfSha256(written))); + Assert.That(cache.GetStored(K)!.StoredVersion, Is.Not.EqualTo(HexOfSha256(slices["k"]))); + } + + // A catalog entry the writer did not choose to replace is carried as the bytes the file holds, a slot this runtime does not + // read included, in either store kind. The ItemId entry is carried: a WsItem stays in the store and only a WsOther is staged. + [Test] + public void ACatalogEntryWithASlotThisRuntimeDoesNotReadSurvivesAWriteOfANeighbour([Values] bool directory, [Values] bool commit) + { + var path = PathOf("catalog-unknown-slot.cc"); + Seed(path, directory, "a"); + File.WriteAllBytes(path, AddAnUnreadSlotToEveryCatalogEntry(File.ReadAllBytes(path))); + var entry = StoreSlices.CatalogEntry(path, ItemId); + using var cache = Open(path, directory); + + Land(cache, commit, typeof(WsOther), new WsOther { Name = "name-x" }, X); + + Assert.That(StoreSlices.CatalogEntry(path, ItemId), Is.EqualTo(entry)); + } + + // The catalog entry for a staged record's id is the writer's, encoded from its object, even where the file publishes another + // entry under that id: a type that adds a compatible id keeps its id, and only the writer's entry carries the change. Here the + // file's ItemId entry differs from the writer's by a slot this runtime does not read, and a WsItem is staged. + [Test] + public void ACatalogEntryForAStagedRecordsIdIsTheWritersEncodingNotTheFilesSlice([Values] bool directory, [Values] bool commit) + { + var path = PathOf("catalog-writer-entry.cc"); + Seed(path, directory, "a"); + var writers = StoreSlices.CatalogEntry(path, ItemId); + File.WriteAllBytes(path, AddAnUnreadSlotToEveryCatalogEntry(File.ReadAllBytes(path))); + Assert.That(StoreSlices.CatalogEntry(path, ItemId), Is.Not.EqualTo(writers)); + using var cache = Open(path, directory); + + Land(cache, commit, typeof(WsItem), new WsItem { Name = "name-x" }, X); + + Assert.That(StoreSlices.CatalogEntry(path, ItemId), Is.EqualTo(writers)); + } + + // The same store with every catalog entry one slot longer than this runtime reads, as a newer writer's would be. + private static byte[] AddAnUnreadSlotToEveryCatalogEntry(byte[] bytes) + { + var output = new List(); + var reader = new MessagePackReader(bytes); + var top = reader.ReadArrayHeader(); + output.Add((byte)(0x90 | top)); + for (var slot = 0; slot < top; slot++) + { + if (slot != 1) + { + var from = (int)reader.Consumed; + reader.Skip(); + output.AddRange(bytes.Skip(from).Take((int)reader.Consumed - from)); + continue; + } + + var count = reader.ReadArrayHeader(); + output.AddRange(new byte[] { 0xdc, (byte)(count >> 8), (byte)count }); + for (var index = 0; index < count; index++) + { + var from = (int)reader.Consumed; + reader.Skip(); + var entry = bytes.Skip(from).Take((int)reader.Consumed - from).ToArray(); + Assert.That(entry[0] & 0xf0, Is.EqualTo(0x90), "a catalog entry is a fixarray"); + output.Add((byte)(entry[0] + 1)); + output.AddRange(entry.Skip(1)); + output.AddRange(new byte[] { 0xa3, (byte)'x', (byte)'y', (byte)'z' }); + } + } + + return output.ToArray(); + } + + private static void OverwriteInPlace(byte[] bytes, string from, string to) + { + var was = System.Text.Encoding.UTF8.GetBytes(from); + var now = System.Text.Encoding.UTF8.GetBytes(to); + Assert.That(now.Length, Is.EqualTo(was.Length)); + var at = IndexOf(bytes, was); + Assert.That(at, Is.GreaterThanOrEqualTo(0), $"'{from}' is in the bytes"); + Array.Copy(now, 0, bytes, at, now.Length); + } + + private static void AddVariant(CultCache cache, string key, CultRecordKey baseKey) => + cache.Commit(batch => batch.UpsertVariant(new CultRecordKey(key), baseKey, new[] + { + cache.Override(nameof(WsItem.Name), "name-" + key), + cache.Override(nameof(WsItem.Note), "variant-" + key) + })); + + // The record under a key now belongs to a schema no registered type claims, as if a build that knows more types wrote it. + private static void MakeForeign(string path, string key) + { + const string foreignId = "sha256:00000000000000000000000000000000000000000000000000000000000050de"; + var snapshot = Read(path); + snapshot.Records.Single(record => record.Key == key).SchemaId = foreignId; + snapshot.SchemaCatalog = snapshot.SchemaCatalog.Append(new CultSchemaCatalogEntry + { + SchemaId = foreignId, + SchemaName = "tests.write_set_foreign", + SchemaVersion = "tests.write_set_foreign.v1", + ContentHash = "foreign", + CanonicalSchemaJson = "{}" + }).ToArray(); + Write(path, snapshot); + } + + // Two writers whose clocks put a record's storedAt ahead of both mint the same storedAt over it. The second to write + // replaces the first's different bytes, so it is stored later than the first's, and the first's copy is told it moved. + // The equal storedAt is the case, and the mint is from the stored one, not from the writer's clock (which is behind it). Either store kind. + [Test] + public void AWriteOverARecordAtItsOwnStoredAtIsStoredLaterWhateverTheClockSays([Values] bool directory, [Values] bool commit) + { + var path = PathOf("skew.cc"); + Seed(path, directory, "k"); + var k = new CultRecordKey("k"); + var ahead = DateTimeOffset.UtcNow.AddHours(1).ToString("O", CultureInfo.InvariantCulture); + // A single file holds the record at that storedAt. A directory store loads a record from its page, which the manifest + // does not own, so there both writers are made to hold it ahead: a stage then mints from it, as a skewed clock does. + if (!directory) + { + var snapshot = Read(path); + snapshot.Records.Single(record => record.Key == "k").StoredAt = ahead; + Write(path, snapshot); + } + + using var a = Open(path, directory); + using var b = Open(path, directory); + if (directory) + { + a.GetStored(k)!.StoredAt = ahead; + b.GetStored(k)!.StoredAt = ahead; + } + + if (commit) + b.Commit(batch => batch.Upsert(typeof(WsItem), new WsItem { Name = "name-k", Note = "by-b" }, k)); + else + { + a.UpsertAsync(typeof(WsItem), new WsItem { Name = "name-k", Note = "by-a" }, k).GetAwaiter().GetResult(); + b.UpsertAsync(typeof(WsItem), new WsItem { Name = "name-k", Note = "by-b" }, k).GetAwaiter().GetResult(); + b.FlushAllBackingStores(); + } + + var bAt = StoredAtOf(path, "k"); + var seenByB = b.Get(k)!; + if (!commit) + Assert.That(a.GetStored(k)!.StoredAt, Is.EqualTo(bAt), "both writers minted the same storedAt over the record"); + + if (commit) + a.Commit(batch => batch.Upsert(typeof(WsItem), new WsItem { Name = "name-k", Note = "by-a" }, k)); + else + a.FlushAllBackingStores(); + + Assert.That(At(StoredAtOf(path, "k")), Is.GreaterThan(At(bAt)), "A's different bytes are stored later than B's"); + Assert.That(b.Commit(batch => + { + batch.Expect(k, seenByB); + batch.Upsert(typeof(WsItem), new WsItem { Name = "name-k", Note = "stale" }, k); + }), Is.False, "B's copy of k is not what the store holds"); + Assert.That(a.Commit(batch => + { + batch.Expect(k, a.Get(k)!); + batch.Upsert(typeof(WsItem), new WsItem { Name = "name-k", Note = "again" }, k); + }), Is.True, "A's own condition holds without a pull: the cache took the storedAt the write gave it"); + } + + // Another writer removes a base and its variant together; this cache, still holding both, adds a variant of that base. + // The base and the variant disappeared from the store, which moved it: the write is refused and the cache reloads. + [Test] + public void AVariantWrittenOverABaseAnotherWriterRemovedIsRefusedAndReloaded() + { + var path = PathOf("gone-base.cc"); + Seed(path, false, "x"); + using var a = Open(path, false); + AddVariant(a, "v", X); + using (var b = Open(path, false)) + Assert.That(b.Commit(batch => { batch.Remove(new CultRecordKey("v")); batch.Remove(X); }), Is.True); + + var conflict = Assert.Throws(() => AddVariant(a, "w", X))!; + + Assert.That(conflict.ChangedKeys, Is.EqualTo(new[] { "v", "x" })); + Assert.That(Keys(path), Is.Empty, "nothing was written"); + Assert.That(a.Get(X), Is.Null, "the cache reloaded what the other writer removed"); + using var fresh = Open(path, false); + Assert.That(fresh.Get(X), Is.Null); + } + + // A foreign refusal whose own reload is refused (dropping x, which a held variant is based on) carries that refusal as its cause. + [Test] + public void AForeignRefusalWhoseReloadFailsCarriesThatCause([Values] bool commit) + { + var path = PathOf("foreign-reload.cc"); + Seed(path, false, "x"); + var a = Open(path, false); + AddVariant(a, "v", X); + MakeForeign(path, "x"); + var before = File.ReadAllBytes(path); + + var refusal = Assert.Throws(() => + Land(a, commit, typeof(WsItem), new WsItem { Name = "name-x", Note = "edited" }, X))!; + + Assert.That(refusal.RecordKey, Is.EqualTo("x")); + Assert.That(refusal.InnerException, Is.TypeOf().And.Message.Contains("variant v is based on it"), "the reload that failed is the cause"); + Assert.That(File.ReadAllBytes(path), Is.EqualTo(before)); + Assert.That(a.Get(X), Is.Not.Null, "the reload failed, so the cache is unchanged"); + } + + // A write that finds a record it holds gone foreign is refused as a conflict, and when the reload that refusal owes fails + // (x cannot go while variant v is based on it) the conflict carries that failure, and nothing changes. + [Test] + public void AWriteConflictWhoseReloadFailsCarriesThatCause([Values] bool commit) + { + var path = PathOf("conflict-reload.cc"); + Seed(path, false, "x"); + var a = Open(path, false); + AddVariant(a, "v", X); + MakeForeign(path, "x"); + var before = File.ReadAllBytes(path); + + var conflict = Assert.Throws(() => + Land(a, commit, typeof(WsItem), new WsItem { Name = "name-y" }, new CultRecordKey("y")))!; + + Assert.That(conflict.InnerException, Is.TypeOf().And.Message.Contains("variant v is based on it"), "the reload that failed is the cause"); + Assert.That(File.ReadAllBytes(path), Is.EqualTo(before)); + Assert.That(a.Get(X), Is.Not.Null, "the reload failed, so the cache is unchanged"); + } + + // A refused foreign record is what the store holds, so the next write into a variant store is not told that it moved. + [Test] + public void AForeignRecordARefusalReloadedIsNotSeenAsMovedByTheNextWrite([Values] bool commit) + { + var path = PathOf("foreign-read.cc"); + Seed(path, false, "a", "x"); + using var a = Open(path, false); + AddVariant(a, "v", A); + MakeForeign(path, "x"); + Assert.Throws(() => + Land(a, commit, typeof(WsItem), new WsItem { Name = "name-x", Note = "edited" }, X)); + + Assert.DoesNotThrow(() => Land(a, commit, typeof(WsItem), new WsItem { Name = "name-n" }, new CultRecordKey("n"))); + + Assert.That(Keys(path), Is.EqualTo(new[] { "a", "n", "v", "x" })); + } + + // A writer that finds the store file gone has read nothing there: a write into the store it makes anew is not refused as + // moved because of what the writer read from the file that was. + [Test] + public void AWriterThatFindsTheStoreFileGoneWritesItAnewWithoutBeingToldItMoved() + { + var path = PathOf("gone-store.cc"); + Seed(path, false, "a"); + using var a = Open(path, false); + AddVariant(a, "v", A); + File.Delete(path); + a.PullAllBackingStoresAsync().GetAwaiter().GetResult(); + + Assert.That(a.Commit(batch => + { + batch.Upsert(typeof(WsItem), new WsItem { Name = "name-a", Note = "again" }, A); + batch.UpsertVariant(new CultRecordKey("v"), A, new[] + { + a.Override(nameof(WsItem.Name), "name-v"), + a.Override(nameof(WsItem.Note), "variant-again") + }); + }), Is.True); + + Assert.That(Keys(path), Is.EqualTo(new[] { "a", "v" })); + } + } +} diff --git a/tests/GameCult.Geometry.Tests/GeometryDocumentTests.cs b/tests/GameCult.Geometry.Tests/GeometryDocumentTests.cs index b5f63775..a58bc29d 100644 --- a/tests/GameCult.Geometry.Tests/GeometryDocumentTests.cs +++ b/tests/GameCult.Geometry.Tests/GeometryDocumentTests.cs @@ -5,6 +5,7 @@ using GameCult.Caching.MessagePack; using GameCult.Mesh; using GameCult.Networking; using MessagePack; +using MessagePack.Resolvers; using NUnit.Framework; using R3; using System; @@ -193,6 +194,19 @@ namespace GameCult.Geometry.Tests .Should().Be("geometry:domain:02c9b5810977406b0c206f3a3494327a423abb9448192be1b9a1863cd0f2ed95"); } + // A source-generated formatter is of no known family and passes the element-id door (ruling c2a-source-generated-formatters-defaulted). + [Test] + public void AGeneratedFormatterPassesTheElementIdDoor() + { + StandardResolver.Instance.GetFormatter().Should() + .NotBeSameAs(DynamicObjectResolver.Instance.GetFormatter(), "the fixture must stay source-generated"); + var payload = File.ReadAllBytes(FixturePath("ragnarok-domain.msgpack")); + var domain = (CultGeometryDomainDocument)CultDocumentMessagePackSerialization.DeserializeUntyped(typeof(CultGeometryDomainDocument), payload); + var again = (CultGeometryDomainDocument)CultDocumentMessagePackSerialization.DeserializeUntyped( + typeof(CultGeometryDomainDocument), CultDocumentMessagePackSerialization.SerializeUntyped(domain, typeof(CultGeometryDomainDocument))); + CultGeometryDomainDocument.CreateRecordKey(again).Value.Should().Be(CultGeometryDomainDocument.CreateRecordKey(domain).Value); + } + [Test] public void RustExportedChunkArtifact_DecodesWithStableCultCacheKey() { diff --git a/tests/GameCult.Mesh.Tests/CultMeshContentVectorTests.cs b/tests/GameCult.Mesh.Tests/CultMeshContentVectorTests.cs index ce9bea2b..da764cc5 100644 --- a/tests/GameCult.Mesh.Tests/CultMeshContentVectorTests.cs +++ b/tests/GameCult.Mesh.Tests/CultMeshContentVectorTests.cs @@ -264,13 +264,18 @@ public sealed class CultMeshContentVectorTests foreach (var depth in new[] { 250, 253, 254, 255, 256, 257, 300 }) { Add($"deep_extra_{depth}", Doc(fields.Append(Deep(depth)))); - AddFields($"deep_chunkextra_{depth}", (7, Arr(Arr(Str(EmptyHash), Raw(0), Raw(0), Str("k"), Deep(depth))))); + AddFields($"deep_chunkextra_{depth}", (7, Arr(Arr(Str(EmptyHash), Raw(0), Raw(0), Str("k"), Str("0"), Deep(depth))))); } AddFields("chunk_3", (7, Arr(Arr(Str(EmptyHash), Raw(0), Raw(0))))); + // Slot 4 is the element id: a string, or nil (the reference reads null); anything else is refused. AddFields("chunk_5", (7, Arr(Arr(Str(EmptyHash), Raw(0), Raw(0), Str("k"), Nil)))); AddFields("chunk_5_nested", (7, Arr(Arr(Str(EmptyHash), Raw(0), Raw(0), Str("k"), Arr(Raw(1), Raw(2)))))); - // An extra slot inside a chunk reference is followed by more bytes, so a skip that stops short shows. + AddFields("chunk_id_str", (7, Arr(Arr(Str(EmptyHash), Raw(0), Raw(0), Str("k"), Str("0"))))); + AddFields("chunk_id_int", (7, Arr(Arr(Str(EmptyHash), Raw(0), Raw(0), Str("k"), Raw(0))))); + AddFields("chunk_id_str8", (7, Arr(Arr(Str(EmptyHash), Raw(0), Raw(0), Str("k"), Raw(0xd9, 1, 0x30))))); + AddFields("chunk_6", (7, Arr(Arr(Str(EmptyHash), Raw(0), Raw(0), Str("k"), Str("0"), Nil)))); + // An extra slot (past the id) inside a chunk reference is followed by more bytes, so a skip that stops short shows. var extras = new (string Name, byte[] Value)[] { ("map", MapOf(Str("a"), Str("b"), Str("c"), Str("d"))), ("map16", Cat(Raw(0xde, 0, 2), Raw(1), Raw(2), Raw(3), Raw(4))), @@ -289,7 +294,7 @@ public sealed class CultMeshContentVectorTests ("nested", Arr(MapOf(Str("a"), Arr(Raw(1), Raw(2))), Arr())) }; foreach (var (name, value) in extras) - AddFields($"chunk_extra_{name}", (7, Arr(Arr(Str(EmptyHash), Raw(0), Raw(0), Str("k"), value)))); + AddFields($"chunk_extra_{name}", (7, Arr(Arr(Str(EmptyHash), Raw(0), Raw(0), Str("k"), Str("0"), value)))); AddFields("chunk_1", (7, Arr(Arr(Str(EmptyHash))))); AddFields("chunk_0", (7, Arr(Arr()))); AddFields("chunk_is_map", (7, Arr(MapOf(Str("a"), Str("b"))))); diff --git a/tests/GameCult.Mesh.Tests/CultMeshSingleFileElementIdTests.cs b/tests/GameCult.Mesh.Tests/CultMeshSingleFileElementIdTests.cs new file mode 100644 index 00000000..a8c501de --- /dev/null +++ b/tests/GameCult.Mesh.Tests/CultMeshSingleFileElementIdTests.cs @@ -0,0 +1,300 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using FluentAssertions; +using GameCult.Caching; +using GameCult.Caching.MessagePack; +using MessagePack; +using NUnit.Framework; + +#nullable enable + +namespace GameCult.Mesh.Tests; + +// C2a: a single-file document written or read through CultMesh is judged as a cache judges it: ids minted, a list that +// cannot hold ids refused, the header decided by what the file holds. +public sealed class CultMeshSingleFileElementIdTests +{ + private const string HexA = "aaaaaaaaaaaa"; + private static readonly CultRecordKey Key = new("publication"); + // A raw write lands beside the typed record, never over it: neither declares the other's schema id. + private static readonly CultRecordKey RawKey = new("raw"); + + private string _root = null!; + + [SetUp] + public void SetUp() + { + _root = Path.Combine(Path.GetTempPath(), "cultmesh-single-file-ids", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(_root); + } + + [TearDown] + public void TearDown() + { + if (Directory.Exists(_root)) Directory.Delete(_root, true); + } + + private static CultMeshBodyPublicationDocument Publication(params CultMeshBodyDescriptor[] representations) => new() + { + BodyId = "aetheria:entities", + ProducerId = "aetheria", + SchemaId = "eve.entity_soa.v1", + Representations = representations + }; + + private static string HeaderOf(string path) => + CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)).FormatVersion; + + private static CultMeshBodyDescriptor Descriptor(string id = "") => new() { Id = id, BodyId = "body" }; + + private string Path_(string name) => Path.Combine(_root, name); + + [Test] + public void TypedWriteMintsTheElementIdsAndMarksTheFileThatHoldsThem() + { + var path = Path_("ids.cc"); + var publication = Publication(Descriptor(), Descriptor()); + + CultMesh.WriteSingleFileDocument(path, Key, publication); + + publication.Representations.Select(entry => entry.Id).Should().OnlyContain(id => id.Length == 12).And.OnlyHaveUniqueItems(); + HeaderOf(path).Should().Be("cultcache.store.v3"); + CultMesh.ReadSingleFileDocument(path, Key).Representations.Select(entry => entry.Id) + .Should().Equal(publication.Representations.Select(entry => entry.Id)); + } + + // A typed write replaces the whole file with a document it can see, so the ids that document holds decide the header. + [Test] + public void TypedWriteDecidesTheHeaderByTheDocumentItWritesEvenOverAMarkedFile() + { + var path = Path_("none.cc"); + CultMesh.WriteSingleFileDocument(path, Key, Publication()); + HeaderOf(path).Should().Be("cultcache.store.v1"); + + CultMesh.WriteSingleFileDocument(path, Key, Publication(Descriptor())); + HeaderOf(path).Should().Be("cultcache.store.v3"); + CultMesh.WriteSingleFileDocument(path, Key, Publication()); + HeaderOf(path).Should().Be("cultcache.store.v1", "the whole file is replaced by a document that holds no id"); + } + + // A raw payload is opaque: the writer cannot see what it holds, so a marked file stays marked. + [Test] + public void RawPayloadWriteOverAMarkedFileKeepsItMarkedAndOverAnUnmarkedOneStaysV1() + { + var path = Path_("raw.cc"); + var schema = new CultMeshSingleFileDocumentSchema("raw:schema", "RawSchema", "1"); + var payload = new byte[] { 0x90 }; + + CultMesh.WriteSingleFileDocumentPayload(path, RawKey, schema, null, payload); + HeaderOf(path).Should().Be("cultcache.store.v1"); + + CultMesh.WriteSingleFileDocument(path, Key, Publication(Descriptor())); + HeaderOf(path).Should().Be("cultcache.store.v3"); + CultMesh.WriteSingleFileDocumentPayload(path, RawKey, schema, null, payload); + HeaderOf(path).Should().Be("cultcache.store.v3", "the payload's ids are not the writer's to see"); + } + + // A raw write at the typed record's own key, by an entry that lists the typed record's id compatible and so may replace it: + // the file then holds only the opaque payload, whose ids the writer cannot see, so the marked file stays marked. (Which files + // a raw write may replace is CultMeshSingleFileReadabilityTests, over the shared vectors.) + [Test] + public void RawPayloadReplacingTheOnlyMarkedRecordKeepsTheFileMarked() + { + var path = Path_("replaced.cc"); + CultMesh.WriteSingleFileDocument(path, Key, Publication(Descriptor())); + HeaderOf(path).Should().Be("cultcache.store.v3"); + var typed = CultDocumentRegistry.Shared.GetRequired().SchemaId; + var schema = new CultMeshSingleFileDocumentSchema("raw:schema", "RawSchema", "1") { CompatibleSchemaIds = new[] { typed } }; + + CultMesh.WriteSingleFileDocumentPayload(path, Key, schema, null, new byte[] { 0x90 }); + + CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)).Records.Should().ContainSingle() + .Which.SchemaId.Should().Be("raw:schema"); + HeaderOf(path).Should().Be("cultcache.store.v3", "the payload's ids are not the writer's to see"); + } + + [Test] + public void TypedWriteRefusesADuplicateIdAndAnElementObjectTwiceAndLeavesTheFileAlone() + { + var path = Path_("refused.cc"); + CultMesh.WriteSingleFileDocument(path, Key, Publication(Descriptor())); + var before = File.ReadAllBytes(path); + + var duplicate = Publication(Descriptor(HexA), Descriptor(HexA)); + var refusal = Assert.Throws(() => CultMesh.WriteSingleFileDocument(path, Key, duplicate))!; + refusal.ElementId.Should().Be(HexA); + + var once = Descriptor(); + var twice = Publication(once, once); + Assert.Throws(() => CultMesh.WriteSingleFileDocument(path, Key, twice))!.Message.Should().Contain("twice"); + once.Id.Should().BeEmpty("a refusal mints nothing"); + + File.ReadAllBytes(path).Should().Equal(before); + } + + [Test] + public void AFailedTypedWriteGivesTheMintedIdsBack() + { + var blocked = Path_("blocked.cc"); + Directory.CreateDirectory(blocked); + var element = Descriptor(); + + Assert.That(() => CultMesh.WriteSingleFileDocument(blocked, Key, Publication(element)), Throws.Exception); + + element.Id.Should().BeEmpty("a write that does not land gives its ids back"); + } + + [Test] + public void TypedReadRefusesWhatALoadRefusesAndMintsWhatALoadMints() + { + var path = Path_("disk.cc"); + CultMesh.WriteSingleFileDocument(path, Key, Publication(Descriptor())); + var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + + var duplicate = Publication(Descriptor(HexA), Descriptor(HexA)); + snapshot.Records[0].Payload = MessagePackSerializer.Serialize(duplicate); + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + var refusal = Assert.Throws(() => CultMesh.ReadSingleFileDocument(path, Key))!; + refusal.ElementId.Should().Be(HexA); + + // A payload written before ids: two reads, and a cache load, agree on the ids minted. + snapshot.Records[0].Payload = MessagePackSerializer.Serialize(Publication(Descriptor(), Descriptor())); + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + var first = CultMesh.ReadSingleFileDocument(path, Key).Representations.Select(entry => entry.Id).ToArray(); + first.Should().OnlyContain(id => id.Length == 12); + CultMesh.ReadSingleFileDocument(path, Key).Representations.Select(entry => entry.Id).Should().Equal(first); + using var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions + { + Registry = CultDocumentRegistry.ForTypes(new[] { typeof(CultMeshBodyPublicationDocument) }), + ReadOnly = true + }); + cache.Get(Key)!.Representations.Select(entry => entry.Id).Should().Equal(first); + } + + // A list document defined in this assembly: the record another writer leaves holding ids under an unmarked header. + [CultDocument("mesh.test.id_list", "mesh.test.id_list.v1")] + [MessagePackObject] + public sealed class MeshIdList + { + [Key(0)] [CultName] public string Name { get; set; } = ""; + [Key(1)] public List Items { get; set; } = new(); + } + + [MessagePackObject] + public sealed class MeshIdItem + { + [Key(0)] public string N { get; set; } = ""; + [Key(1)] [CultElementId] public string Id { get; set; } = ""; + } + + private static readonly CultRecordKey IdsKey = new("ids"); + private const string HexB = "bbbbbbbbbbbb"; + + // A v1 file whose record holds 12-hex element ids: the document is written with the type (v3), then the header is put back + // to the v1 a writer that does not decide would have left. + private string UnmarkedStoreHoldingIds() + { + var path = Path_("unmarked.cc"); + CultMesh.WriteSingleFileDocument( + path, + IdsKey, + new MeshIdList { Name = "ids", Items = { new MeshIdItem { Id = HexA }, new MeshIdItem { Id = HexB } } }, + registry: Registry(typeof(MeshIdList))); + var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + snapshot.FormatVersion = CultPersistedStoreSnapshot.FormatV1; + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + HeaderOf(path).Should().Be("cultcache.store.v1"); + return path; + } + + private string PythonStore() + { + string? vectors = null; + for (var dir = new DirectoryInfo(TestContext.CurrentContext.TestDirectory); dir != null && vectors == null; dir = dir.Parent) + { + var candidate = Path.Combine(dir.FullName, "tests", "vectors", "numeric-leaves", "py-store", "leaf-record.cc"); + if (File.Exists(candidate)) vectors = candidate; + } + + vectors.Should().NotBeNull("the shared vectors sit above the test directory"); + var path = Path_("leaf-record.cc"); + File.Copy(vectors!, path); + HeaderOf(path).Should().Be("cultcache.store.v1", "cultcache-py leaves the header unmarked"); + return path; + } + + private static CultDocumentRegistry Registry(params Type[] types) => CultDocumentRegistry.ForTypes(types); + + // A CultMesh write is a writer with a type: it decides for the copied record from the bytes the file holds, whatever any + // earlier read saw (a fresh store per call has read nothing). + [Test] + public void ACultMeshWriteThatCopiesARecordHoldingIdsMarksTheStore() + { + var path = UnmarkedStoreHoldingIds(); + + CultMesh.WriteSingleFileDocument(path, Key, Publication(), registry: Registry(typeof(MeshIdList), typeof(CultMeshBodyPublicationDocument))); + + HeaderOf(path).Should().Be("cultcache.store.v3"); + CultMesh.ReadSingleFileDocument(path, IdsKey, Registry(typeof(MeshIdList))).Items.Select(item => item.Id).Should().Equal(HexA, HexB); + } + + // A raw payload is opaque by design: it attaches no registry and decides nothing, even beside a record it could have read. + [Test] + public void ARawCultMeshWriteBesideARecordHoldingIdsLeavesTheHeaderAlone() + { + var path = UnmarkedStoreHoldingIds(); + + CultMesh.WriteSingleFileDocumentPayload(path, RawKey, new CultMeshSingleFileDocumentSchema("raw:schema", "RawSchema", "1"), null, new byte[] { 0x90 }); + + HeaderOf(path).Should().Be("cultcache.store.v1"); + } + + // Soul s1 P8, committed: the writer's registry has only the type it writes, so the python record is foreign to it and it + // decides nothing for it (variants:question:non-csharp-writers-mark-ids-2). + [Test] + public void ACultMeshWriteWhoseRegistryLacksTheRecordsTypeDecidesNothingForIt() + { + var path = PythonStore(); + + CultMesh.WriteSingleFileDocument(path, Key, Publication(), registry: Registry(typeof(CultMeshBodyPublicationDocument))); + + HeaderOf(path).Should().Be("cultcache.store.v1"); + } + + // The copy check never refuses a write: a record the registry claims by schema name but cannot resolve (the catalog entry + // names another type for a slot) is copied as its bytes and decides nothing. Soul s2 P5, incompatible-catalog-slot12. + [Test] + public void AnUnresolvableCopiedRecordIsCopiedAndDecidesNothing() + { + var path = UnmarkedStoreHoldingIds(); + var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + const string changedId = "mesh.test.id_list.v0"; + var entry = snapshot.SchemaCatalog.Single(candidate => candidate.SchemaName == "mesh.test.id_list"); + entry.SchemaId = changedId; + entry.ContentHash = "persisted-by-another-runtime"; + entry.Members.Single(member => member.Slot == 1).TypeName = "System.Collections.Generic.List`1[[Other.Inner]]"; + snapshot.Records.Single(record => record.Key == IdsKey.Value).SchemaId = changedId; + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); + var before = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)).Records.Single(record => record.Key == IdsKey.Value); + + Assert.DoesNotThrow(() => CultMesh.WriteSingleFileDocument(path, Key, Publication(), registry: Registry(typeof(MeshIdList), typeof(CultMeshBodyPublicationDocument)))); + + var after = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)).Records.Single(record => record.Key == IdsKey.Value); + after.SchemaId.Should().Be(changedId); + after.Payload.Should().Equal(before.Payload); + HeaderOf(path).Should().Be("cultcache.store.v1"); + } + + // Typed writes beside records that hold no id leave the store unmarked: the check marks only on an id it found. + [Test] + public void ATypedWriteBesideARecordThatHoldsNoIdKeepsTheStoreUnmarked() + { + var path = Path_("plain.cc"); + CultMesh.WriteSingleFileDocument(path, Key, Publication(), registry: Registry(typeof(CultMeshBodyPublicationDocument))); + CultMesh.WriteSingleFileDocument(path, new CultRecordKey("second"), Publication(), registry: Registry(typeof(CultMeshBodyPublicationDocument))); + + HeaderOf(path).Should().Be("cultcache.store.v1"); + } +} diff --git a/tests/GameCult.Mesh.Tests/CultMeshSingleFileReadabilityTests.cs b/tests/GameCult.Mesh.Tests/CultMeshSingleFileReadabilityTests.cs new file mode 100644 index 00000000..556d785d --- /dev/null +++ b/tests/GameCult.Mesh.Tests/CultMeshSingleFileReadabilityTests.cs @@ -0,0 +1,122 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using FluentAssertions; +using GameCult.Caching; +using GameCult.Caching.MessagePack; +using NUnit.Framework; + +#nullable enable + +namespace GameCult.Mesh.Tests; + +// A single-file write replaces a file exactly when the runtime's own reader opens it, as a cache's flush does. The bytes and +// the verdict are shared with every runtime: tests/vectors/document-variants-c2a/readability. +public sealed class CultMeshSingleFileReadabilityTests +{ + private const int CSharp = 1; + private string _root = null!; + + [SetUp] + public void SetUp() + { + _root = Path.Combine(Path.GetTempPath(), "cultmesh-single-file-readability", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(_root); + } + + [TearDown] + public void TearDown() + { + if (Directory.Exists(_root)) Directory.Delete(_root, true); + } + + private static string VectorRoot() + { + for (var dir = new DirectoryInfo(TestContext.CurrentContext.TestDirectory); dir != null; dir = dir.Parent) + { + var candidate = Path.Combine(dir.FullName, "tests", "vectors", "document-variants-c2a", "readability"); + if (Directory.Exists(candidate)) return candidate; + } + + throw new DirectoryNotFoundException($"Shared readability vectors not found above {TestContext.CurrentContext.TestDirectory}."); + } + + public static IEnumerable Vectors() => + File.ReadAllLines(Path.Combine(VectorRoot(), "manifest.txt")) + .Where(line => line.Length > 0 && line[0] != '#') + .Select(line => line.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries)) + .Select(cells => new TestCaseData(cells[0], cells[CSharp] == "reads").SetName($"{cells[0]} {cells[CSharp]}")); + + private static CultMeshBodyPublicationDocument Publication() => new() + { + BodyId = "aetheria:entities", + ProducerId = "aetheria", + SchemaId = "eve.entity_soa.v1" + }; + + [TestCaseSource(nameof(Vectors))] + public void AWriteReplacesAFileExactlyWhenItReads(string vector, bool reads) + { + var raw = new CultMeshSingleFileDocumentSchema("raw:schema", "RawSchema", "1"); + var writes = new (string Name, Action Write)[] + { + ("typed", path => CultMesh.WriteSingleFileDocument(path, new CultRecordKey("publication"), Publication())), + ("raw payload", path => CultMesh.WriteSingleFileDocumentPayload(path, new CultRecordKey("publication"), raw, null, new byte[] { 0x90 })) + }; + foreach (var (name, write) in writes) + { + var path = Path.Combine(_root, name.Replace(' ', '-') + ".cc"); + var bytes = File.ReadAllBytes(Path.Combine(VectorRoot(), vector)); + File.WriteAllBytes(path, bytes); + + if (reads) + { + var before = CultDocumentMessagePackSerialization.DeserializeSnapshot(bytes); + if (before.Records.Any(record => record.Variant != null)) + { + Assert.That(() => write(path), Throws.TypeOf(), name); + File.ReadAllBytes(path).Should().Equal(bytes, name + " wrote into a store holding a variant it cannot resolve"); + continue; + } + + write(path); + var after = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + after.Records.Where(record => record.Key == "publication").Should().ContainSingle(name); + var others = before.Records.Where(record => record.Key != "publication").ToArray(); + after.Records.Should().HaveCount(others.Length + 1, name); + foreach (var record in others) + { + var kept = after.Records.Single(candidate => candidate.Key == record.Key); + kept.SchemaId.Should().Be(record.SchemaId, name); + kept.StoredAt.Should().Be(record.StoredAt, name); + kept.Payload.Should().Equal(record.Payload, name + " copies the file's other records as they are"); + } + + continue; + } + + Assert.That(() => write(path), Throws.TypeOf(), name); + File.ReadAllBytes(path).Should().Equal(bytes, name + " rewrote a file this runtime cannot read"); + } + } + + // A read asks the same reader as a write: a file this runtime cannot read, a zero-byte one included, is refused as unreadable + // and never reported as a store that lacks the record. + [Test] + public void AReadOfAZeroByteFileIsRefusedAsUnreadableNotAsAMissingRecord() + { + var path = Path.Combine(_root, "zero.cc"); + File.WriteAllBytes(path, Array.Empty()); + + Assert.That(() => CultMesh.ReadSingleFileDocumentPayload(path, new CultRecordKey("publication"), "raw:schema"), Throws.TypeOf()); + } + + [Test] + public void EveryVectorInTheFolderHasAManifestRow() + { + var rows = Vectors().Select(row => (string)row.Arguments[0]).Where(name => !name.StartsWith("..", StringComparison.Ordinal)).OrderBy(name => name, StringComparer.Ordinal); + var files = Directory.GetFiles(VectorRoot(), "*.bin").Select(Path.GetFileName).OrderBy(name => name, StringComparer.Ordinal); + rows.Should().Equal(files); + } +} diff --git a/tests/GameCult.Mesh.Tests/CultMeshSingleFileStoreTests.cs b/tests/GameCult.Mesh.Tests/CultMeshSingleFileStoreTests.cs new file mode 100644 index 00000000..7ce94ed2 --- /dev/null +++ b/tests/GameCult.Mesh.Tests/CultMeshSingleFileStoreTests.cs @@ -0,0 +1,465 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using FluentAssertions; +using GameCult.Caching; +using GameCult.Caching.MessagePack; +using NUnit.Framework; + +#nullable enable + +namespace GameCult.Mesh.Tests; + +// CultMesh's single-file helpers reach the store file through the one CultCache single-file store: what is at the path, the lock +// and the atomic replace are decided there, once, for a Mesh read or write and a cache open, flush or commit alike. +public sealed class CultMeshSingleFileStoreTests +{ + private static readonly CultRecordKey Key = new("publication"); + private static readonly CultMeshSingleFileDocumentSchema Raw = new("raw:schema", "RawSchema", "1"); + private static readonly byte[] EmptyArray = { 0x90 }; + private static readonly CultCacheOpenOptions ReadOnlyOpen = new() { FlushOnDispose = false, StoreFlushOnDispose = false }; + private string _root = null!; + + [SetUp] + public void SetUp() + { + _root = Path.Combine(Path.GetTempPath(), "cultmesh-single-file-store", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(_root); + } + + [TearDown] + public void TearDown() + { + if (!Directory.Exists(_root)) return; + // A recursive delete cannot remove a link whose target is gone; take the links out by themselves first. + foreach (var entry in Directory.EnumerateFileSystemEntries(_root).Where(IsALink)) + { + if ((new FileInfo(entry).Attributes & FileAttributes.Directory) != 0) Directory.Delete(entry, false); + else File.Delete(entry); + } + + Directory.Delete(_root, true); + } + + private static CultMeshBodyPublicationDocument Publication(string producer = "aetheria") => new() + { + BodyId = "aetheria:entities", + ProducerId = producer, + SchemaId = "eve.entity_soa.v1" + }; + + private static CultMeshBodyPublicationDocument PublicationOf(string id) => new() + { + BodyId = "aetheria:" + id, + ProducerId = "aetheria", + SchemaId = "eve.entity_soa.v1" + }; + + private static string SchemaId() => CultDocumentRegistry.Shared.GetRequired().SchemaId; + + private static bool IsALink(string path) + { + var attributes = new FileInfo(path).Attributes; + return (int)attributes != -1 && (attributes & FileAttributes.ReparsePoint) != 0; + } + + private static int RecordsSeenByACache(string path) + { + using var cache = CultCacheMessagePack.Create(path, ReadOnlyOpen); + return cache.GetAll().Count(); + } + + private IEnumerable TempFiles() => + Directory.GetFileSystemEntries(_root).Where(entry => entry.EndsWith(".tmp", StringComparison.Ordinal)); + + // A link whose target is gone: a symlink where one can be made, a directory junction on Windows, which needs no privilege. + private static void CreateDanglingLink(string path, string target) + { + if (!OperatingSystem.IsWindows()) + { + File.CreateSymbolicLink(path, target); + return; + } + + Directory.CreateDirectory(target); + var mklink = Process.Start(new ProcessStartInfo("cmd.exe", $"/c mklink /J \"{path}\" \"{target}\"") + { + CreateNoWindow = true, + RedirectStandardOutput = true, + RedirectStandardError = true, + })!; + mklink.WaitForExit(); + Assert.That(mklink.ExitCode, Is.Zero, "mklink /J failed"); + Directory.Delete(target); + } + + private static void CreateLiveLinkOrIgnore(string path, string target) + { + try + { + File.CreateSymbolicLink(path, target); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + Assert.Ignore("A file symlink needs a privilege this account lacks: " + ex.Message); + } + } + + private static byte[] EntryBytes(CultPersistedStoreSnapshot snapshot, string schemaId) => + CultDocumentMessagePackSerialization.SerializeSnapshot(new CultPersistedStoreSnapshot + { + SchemaCatalog = snapshot.SchemaCatalog.Where(entry => entry.SchemaId == schemaId).ToArray() + }); + + private static CultPersistedStoreSnapshot Snapshot(string path) => + CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + + // A Mesh write is a write set of one key: every other record in the file and the entry that publishes it are copied forward as + // the file holds them, and a second write of the key replaces only that key. + [Test] + public void AMeshWriteIntoAStoreHoldingOtherRecordsCopiesThemForwardAndReplacesOnlyItsKey() + { + var path = Path.Combine(_root, "shared.cc"); + using (var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions())) + { + cache.Commit(batch => + { + batch.Upsert(typeof(CultMeshBodyPublicationDocument), PublicationOf("a"), new CultRecordKey("a")); + batch.Upsert(typeof(CultMeshBodyPublicationDocument), PublicationOf("b"), new CultRecordKey("b")); + }); + } + + var before = Snapshot(path); + CultMesh.WriteSingleFileDocumentPayload(path, Key, Raw, null, new byte[] { 0x90 }); + var after = Snapshot(path); + + after.Records.Select(record => record.Key).Should().Equal("a", "b", "publication"); + foreach (var key in new[] { "a", "b" }) + { + var was = before.Records.Single(record => record.Key == key); + var kept = after.Records.Single(record => record.Key == key); + (kept.SchemaId, kept.StoredAt).Should().Be((was.SchemaId, was.StoredAt)); + kept.Payload.Should().Equal(was.Payload); + } + + EntryBytes(after, SchemaId()).Should().Equal(EntryBytes(before, SchemaId())); + after.SchemaCatalog.Should().Contain(entry => entry.SchemaId == "raw:schema"); + + CultMesh.WriteSingleFileDocumentPayload(path, Key, Raw, null, new byte[] { 0x91, 0x01 }); + var again = Snapshot(path); + again.Records.Select(record => record.Key).Should().Equal("a", "b", "publication"); + again.Records.Single(record => record.Key == "publication").Payload.Should().Equal(new byte[] { 0x91, 0x01 }); + foreach (var key in new[] { "a", "b" }) + again.Records.Single(record => record.Key == key).Payload.Should().Equal(before.Records.Single(record => record.Key == key).Payload); + } + + // A Mesh write has read nothing and has no cache to resolve a variant against its base, so it never writes into a store that holds + // one: typed or raw, at the variant's key, its base's key or an unrelated key, it is refused and the file is left byte for byte. + [Test] + public void AMeshWriteIntoAStoreHoldingAVariantIsRefusedAndLeavesItByteIdentical( + [Values("variant", "base", "unrelated")] string target, [Values] bool typed) + { + var path = Path.Combine(_root, $"variant-{target}-{typed}.cc"); + var baseKey = new CultRecordKey("a"); + using (var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions())) + { + cache.Commit(batch => + { + batch.Upsert(typeof(CultMeshBodyPublicationDocument), PublicationOf("a"), baseKey); + batch.UpsertVariant(new CultRecordKey("v"), baseKey, new[] + { + cache.Override(nameof(CultMeshBodyPublicationDocument.BodyId), "aetheria:v"), + cache.Override(nameof(CultMeshBodyPublicationDocument.ProducerId), "variant-producer") + }); + }); + } + + var bytes = File.ReadAllBytes(path); + Snapshot(path).Records.Should().Contain(record => record.Variant != null); + var key = new CultRecordKey(target switch { "variant" => "v", "base" => "a", _ => "z" }); + + void Write() + { + if (typed) + CultMesh.WriteSingleFileDocument(path, key, PublicationOf("z")); + else + CultMesh.WriteSingleFileDocumentPayload(path, key, Raw, null, new byte[] { 0x90 }); + } + + // A raw write at the variant's or the base's key names a record under an id its entry does not declare, and that refusal is + // checked first. + if (!typed && target != "unrelated") + Assert.Throws(Write)!.RecordKey.Should().Be(key.Value); + else + Assert.Throws(Write)!.Message.Should().Contain("CultCache"); + File.ReadAllBytes(path).Should().Equal(bytes); + } + + // A Mesh write declares the ids of the catalog entry it writes: its own and those it lists compatible. Over a record under any + // other id, typed or raw, it is refused with the typed schema conflict naming the record, and the file is left byte for byte. + [Test] + public void AMeshWriteOverARecordUnderAnIdItsEntryDoesNotDeclareIsRefused([Values] bool typed) + { + var path = Path.Combine(_root, $"undeclared-{typed}.cc"); + if (typed) + CultMesh.WriteSingleFileDocumentPayload(path, Key, Raw, null, EmptyArray); + else + CultMesh.WriteSingleFileDocument(path, Key, Publication()); + var bytes = File.ReadAllBytes(path); + + var refusal = Assert.Throws(() => + { + if (typed) + CultMesh.WriteSingleFileDocument(path, Key, Publication("other")); + else + CultMesh.WriteSingleFileDocumentPayload(path, Key, Raw, null, new byte[] { 0x91, 0x01 }); + })!; + + (refusal.RecordKey, refusal.SchemaId).Should().Be((Key.Value, typed ? Raw.SchemaId : SchemaId())); + refusal.Message.Should().Contain("nothing was reloaded"); + File.ReadAllBytes(path).Should().Equal(bytes); + } + + // A write whose entry lists the held id compatible owns the record and replaces it under its own id. + [Test] + public void AMeshWriteWhoseEntryListsTheHeldIdLandsUnderItsOwnId() + { + var path = Path.Combine(_root, "listed.cc"); + CultMesh.WriteSingleFileDocumentPayload(path, Key, Raw, null, EmptyArray); + var next = new CultMeshSingleFileDocumentSchema("raw:schema.v2", "RawSchema", "2") { CompatibleSchemaIds = new[] { Raw.SchemaId } }; + + CultMesh.WriteSingleFileDocumentPayload(path, Key, next, null, new byte[] { 0x91, 0x01 }); + + var record = Snapshot(path).Records.Single(); + (record.Key, record.SchemaId).Should().Be((Key.Value, "raw:schema.v2")); + record.Payload.Should().Equal(new byte[] { 0x91, 0x01 }); + } + + // The writer cannot see whether the raw payload or the records it copies hold element ids, so a marked file stays marked. + [Test] + public void AMeshRawWriteIntoAMarkedStoreKeepsItMarked() + { + var path = Path.Combine(_root, "marked.cc"); + using (var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions())) + cache.Commit(batch => batch.Upsert(typeof(CultMeshBodyPublicationDocument), PublicationOf("a"), new CultRecordKey("a"))); + var marked = Snapshot(path); + marked.FormatVersion = CultPersistedStoreSnapshot.FormatV3; + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(marked)); + + CultMesh.WriteSingleFileDocumentPayload(path, Key, Raw, null, new byte[] { 0x90 }); + + Snapshot(path).FormatVersion.Should().Be(CultPersistedStoreSnapshot.FormatV3); + } + + [Test] + public void ADanglingLinkIsAnIoErrorOnTypedWriteRawWriteAndReadNeverAMissingFile() + { + var path = Path.Combine(_root, "dangling.cc"); + var target = Path.Combine(_root, "unmounted"); + CreateDanglingLink(path, target); + + var operations = new (string Name, Action Run)[] + { + ("typed write", () => CultMesh.WriteSingleFileDocument(path, Key, Publication())), + ("raw write", () => CultMesh.WriteSingleFileDocumentPayload(path, Key, Raw, null, EmptyArray)), + ("payload read", () => CultMesh.ReadSingleFileDocumentPayload(path, Key, "raw:schema")), + ("keyless payload read", () => CultMesh.ReadSingleFileDocumentPayload(path, "raw:schema")), + ("typed read", () => CultMesh.ReadSingleFileDocument(path, Key)), + }; + foreach (var (name, run) in operations) + { + var error = Assert.Catch(() => run(), name)!; + error.Should().NotBeOfType(name + " reported a dangling link as a missing file"); + } + + IsALink(path).Should().BeTrue("the link was replaced"); + Directory.Exists(target).Should().BeFalse("a write went through the link"); + File.Exists(target).Should().BeFalse("a write went through the link"); + TempFiles().Should().BeEmpty(); + } + + [Test] + public void OnlyNothingAtThePathIsAMissingFile() + { + var path = Path.Combine(_root, "missing.cc"); + var missing = Assert.Throws(() => CultMesh.ReadSingleFileDocumentPayload(path, Key, "raw:schema"))!; + missing.FileName.Should().Be(path); + Assert.Throws(() => CultMesh.ReadSingleFileDocumentPayload(path, "raw:schema")); + Assert.Throws(() => CultMesh.ReadSingleFileDocument(path, Key)); + File.Exists(path).Should().BeFalse("a read created the file"); + + // A write to nothing at the path creates the store, and the reader reads it back. + CultMesh.WriteSingleFileDocumentPayload(path, Key, Raw, null, EmptyArray); + CultMesh.ReadSingleFileDocumentPayload(path, Key, "raw:schema").Should().Equal(EmptyArray); + } + + [Test] + public void ADirectoryAtThePathIsAnIoErrorOnReadAndWriteAndIsLeftAlone() + { + var path = Path.Combine(_root, "directory.cc"); + Directory.CreateDirectory(path); + + var operations = new (string Name, Action Run)[] + { + ("typed write", () => CultMesh.WriteSingleFileDocument(path, Key, Publication())), + ("raw write", () => CultMesh.WriteSingleFileDocumentPayload(path, Key, Raw, null, EmptyArray)), + ("payload read", () => CultMesh.ReadSingleFileDocumentPayload(path, Key, "raw:schema")), + ("keyless payload read", () => CultMesh.ReadSingleFileDocumentPayload(path, "raw:schema")), + }; + foreach (var (name, run) in operations) + { + var error = Assert.Catch(() => run(), name)!; + error.Should().NotBeOfType(name); + error.Message.Should().Contain("directory", name); + error.Message.Should().NotContain(_root, name + " echoed the path in its refusal text"); + } + + Directory.Exists(path).Should().BeTrue(); + Directory.GetFileSystemEntries(path).Should().BeEmpty(); + TempFiles().Should().BeEmpty(); + } + + [Test] + public void AZeroByteFileIsRefusedOnTypedAndRawWriteAndLeftAlone() + { + var path = Path.Combine(_root, "zero.cc"); + File.WriteAllBytes(path, Array.Empty()); + + Assert.Throws(() => CultMesh.WriteSingleFileDocument(path, Key, Publication())); + Assert.Throws(() => CultMesh.WriteSingleFileDocumentPayload(path, Key, Raw, null, EmptyArray)); + Assert.Throws(() => CultMesh.ReadSingleFileDocument(path, Key)); + new FileInfo(path).Length.Should().Be(0); + } + + [Test] + public void AMeshWriteWaitsForTheStoreLock() + { + var path = Path.Combine(_root, "locked.cc"); + FileStream? held = new FileStream(path + ".lock", FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None); + try + { + var write = Task.Run(() => CultMesh.WriteSingleFileDocumentPayload(path, Key, Raw, null, EmptyArray)); + + write.Wait(TimeSpan.FromMilliseconds(300)).Should().BeFalse("a Mesh write went ahead of the store lock"); + File.Exists(path).Should().BeFalse("a Mesh write landed while the lock was held"); + + held.Dispose(); + held = null; + write.Wait(TimeSpan.FromSeconds(10)).Should().BeTrue("the write never completed after the lock was released"); + write.IsCompletedSuccessfully.Should().BeTrue(); + } + finally + { + held?.Dispose(); + } + + CultMesh.ReadSingleFileDocumentPayload(path, Key, "raw:schema").Should().Equal(EmptyArray); + } + + [Test] + public void ACacheOpeningDuringMeshWritesNeverSeesAnEmptyStore() + { + // Windows replaces with ReplaceFile, which leaves a moment with nothing at the path for a lock-free reader: 1,722 empty + // stores in 17,313 opens at b1ff2dcb. That is the store's replace, not Mesh's, and it is the same for a cache flush beside a + // cache open; the fork is in the cut report. + if (OperatingSystem.IsWindows()) + Assert.Ignore("Windows File.Replace is not atomic to a lock-free reader; see cut report fork mesh-reader-windows-replace-window."); + + var path = Path.Combine(_root, "race.cc"); + CultMesh.WriteSingleFileDocument(path, Key, Publication()); + const int writes = 500; + var stop = 0; + var errors = new ConcurrentQueue(); + var writer = Task.Run(() => + { + for (var index = 0; index < writes; index++) + { + try { CultMesh.WriteSingleFileDocument(path, Key, Publication()); } + catch (Exception ex) { errors.Enqueue("write: " + ex.GetType().Name + ": " + ex.Message); } + } + + Volatile.Write(ref stop, 1); + }); + + int opens = 0, empty = 0, reads = 0; + var clock = Stopwatch.StartNew(); + while (Volatile.Read(ref stop) == 0 && clock.Elapsed < TimeSpan.FromSeconds(60)) + { + try + { + opens++; + if (RecordsSeenByACache(path) == 0) empty++; + CultMesh.ReadSingleFileDocumentPayload(path, Key, SchemaId()); + reads++; + } + catch (Exception ex) + { + errors.Enqueue("read: " + ex.GetType().Name + ": " + ex.Message); + } + } + + writer.Wait(TimeSpan.FromSeconds(60)).Should().BeTrue(); + TestContext.Out.WriteLine($"mesh writes={writes} cache opens={opens} empty stores={empty} mesh reads ok={reads} errors={errors.Count}"); + errors.Take(3).Should().BeEmpty(); + empty.Should().Be(0, "a cache opened in the window where nothing was at the path"); + opens.Should().BeGreaterThan(0); + } + + [Test] + public void ConcurrentMeshWritersNeverErrorAndLeaveAReadableFile() + { + var path = Path.Combine(_root, "writers.cc"); + const int writers = 4, each = 100; + var errors = new ConcurrentQueue(); + var tasks = Enumerable.Range(0, writers).Select(writer => Task.Run(() => + { + for (var index = 0; index < each; index++) + { + try { CultMesh.WriteSingleFileDocument(path, Key, Publication("writer-" + writer)); } + catch (Exception ex) { errors.Enqueue(ex.GetType().Name + ": " + ex.Message); } + } + })).ToArray(); + + Task.WaitAll(tasks, TimeSpan.FromSeconds(120)).Should().BeTrue(); + TestContext.Out.WriteLine($"mesh writers={writers} writes each={each} errors={errors.Count}"); + errors.Take(3).Should().BeEmpty(); + File.Exists(path).Should().BeTrue("concurrent writers deleted the store"); + CultMesh.ReadSingleFileDocument(path, Key).ProducerId.Should().StartWith("writer-"); + RecordsSeenByACache(path).Should().Be(1); + TempFiles().Should().BeEmpty(); + } + + // The store's replace decides what a write leaves where a live link was, and a Mesh write leaves what a cache commit leaves: + // one replace, whatever the platform does with a link. + [Test] + public void AMeshWriteOverALiveLinkLeavesWhatACacheCommitLeaves() + { + string Outcome(string name, Action write) + { + var target = Path.Combine(_root, name + "-target.cc"); + var link = Path.Combine(_root, name + "-link.cc"); + CultMesh.WriteSingleFileDocument(target, Key, Publication("first")); + CreateLiveLinkOrIgnore(link, target); + var before = File.ReadAllBytes(target); + + write(link); + + CultMesh.ReadSingleFileDocument(link, Key).ProducerId.Should().Be("second", name); + return $"linkKept={IsALink(link)} targetChanged={!File.ReadAllBytes(target).SequenceEqual(before)}"; + } + + var viaMesh = Outcome("mesh", link => CultMesh.WriteSingleFileDocument(link, Key, Publication("second"))); + var viaCache = Outcome("cache", link => + { + using var cache = CultCacheMessagePack.Create(link, ReadOnlyOpen); + cache.Commit(batch => batch.Upsert(typeof(CultMeshBodyPublicationDocument), Publication("second"), Key)); + }); + + viaMesh.Should().Be(viaCache); + } +} diff --git a/tests/GameCult.Mesh.Tests/CultMeshSingleFileWriteGuardTests.cs b/tests/GameCult.Mesh.Tests/CultMeshSingleFileWriteGuardTests.cs index 4312e0cb..e1cb1cb5 100644 --- a/tests/GameCult.Mesh.Tests/CultMeshSingleFileWriteGuardTests.cs +++ b/tests/GameCult.Mesh.Tests/CultMeshSingleFileWriteGuardTests.cs @@ -9,8 +9,9 @@ using NUnit.Framework; namespace GameCult.Mesh.Tests; -// A CultMesh single-file write replaces the whole file, so it must read what it replaces first. A store stamped with a -// newer format version, one it cannot read, or one holding a record the write would drop is refused and left byte-identical. +// A CultMesh single-file write goes through the cache's single-file store: it reads the file under the store lock, applies its +// one record, and keeps every other record. A store stamped with a newer format version or one it cannot read is refused and +// left byte-identical. public sealed class CultMeshSingleFileWriteGuardTests { private static readonly CultMeshSingleFileDocumentSchema Schema = new("sha256:guard", "guard.item", "1"); @@ -34,7 +35,7 @@ public sealed class CultMeshSingleFileWriteGuardTests private void Write(string key, byte payload) => CultMesh.WriteSingleFileDocumentPayload(_path, new CultRecordKey(key), Schema, null, new[] { payload }); - [TestCase("cultcache.store.v3")] + [TestCase("cultcache.store.v4")] [TestCase("cultcache.store.v9")] public void WriteRefusesAStoreStampedWithANewerFormat(string newer) { @@ -44,7 +45,7 @@ public sealed class CultMeshSingleFileWriteGuardTests File.WriteAllBytes(_path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); var before = File.ReadAllBytes(_path); - var error = Assert.Throws(() => Write("item:a", 2))!; + var error = Assert.Throws(() => Write("item:a", 2))!; error.Message.Should().Contain(newer); File.ReadAllBytes(_path).Should().Equal(before); @@ -61,16 +62,18 @@ public sealed class CultMeshSingleFileWriteGuardTests File.ReadAllBytes(_path).Should().Equal(garbage); } + // Another writer's record, under a schema this write does not touch, is still in the file after the write. [Test] - public void WriteRefusesToDropAnotherRecord() + public void WriteKeepsAnotherWritersRecord() { + var other = new CultMeshSingleFileDocumentSchema("sha256:other", "other.item", "1"); Write("item:a", 1); - var before = File.ReadAllBytes(_path); + CultMesh.WriteSingleFileDocumentPayload(_path, new CultRecordKey("item:b"), other, null, new byte[] { 7 }); - var error = Assert.Throws(() => Write("item:b", 2))!; + Write("item:a", 2); - error.Message.Should().Contain("item:a"); - File.ReadAllBytes(_path).Should().Equal(before); + CultMesh.ReadSingleFileDocumentPayload(_path, new CultRecordKey("item:a"), Schema.SchemaId).Should().Equal(2); + CultMesh.ReadSingleFileDocumentPayload(_path, new CultRecordKey("item:b"), other.SchemaId).Should().Equal(7); } [Test] @@ -112,7 +115,7 @@ public sealed class CultMeshSingleFileWriteGuardTests File.WriteAllBytes(_path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); var before = File.ReadAllBytes(_path); - Assert.Throws(() => Write("item:a", 2)); + Assert.Throws(() => Write("item:a", 2)); File.ReadAllBytes(_path).Should().Equal(before); } diff --git a/tests/GameCult.Mesh.Tests/CultMeshStreamingTests.cs b/tests/GameCult.Mesh.Tests/CultMeshStreamingTests.cs index 63747c29..5a9838c8 100644 --- a/tests/GameCult.Mesh.Tests/CultMeshStreamingTests.cs +++ b/tests/GameCult.Mesh.Tests/CultMeshStreamingTests.cs @@ -696,7 +696,7 @@ public sealed class CultMeshStreamingTests } [Test] - public async Task DocumentHandle_ExposesTypedSnapshotsAndSameSchemaAliases() + public async Task DocumentHandle_ExposesTypedSnapshotsAndWatches() { var subject = new Subject(); var route = new CultMeshRouteHint(CultMeshLocalityKind.SharedMemory, "co-located document slab"); @@ -733,17 +733,8 @@ public sealed class CultMeshStreamingTests snapshot.Text.Should().Be("primary"); handle.Latest().Text.Should().Be("primary"); - var alias = handle.AsSchemaAlias(); - var aliasSnapshot = await alias.LatestAsync(); - alias.Latest().Text.Should().Be("primary"); - alias.DocumentId.Should().Be(handle.DocumentId); - alias.SchemaName.Should().Be(handle.SchemaName); - alias.SchemaVersion.Should().Be(handle.SchemaVersion); - aliasSnapshot.Text.Should().Be("primary"); - aliasSnapshot.Revision.Should().Be(1); - - MeshNoteAliasDocument observed = null!; - using var subscription = alias.Watch(value => observed = value); + MeshNoteDocument observed = null!; + using var subscription = handle.Watch(value => observed = value); subject.OnNext(new MeshNoteDocument { Schema = "tests.mesh_note.v1", @@ -790,28 +781,7 @@ public sealed class CultMeshStreamingTests } [Test] - public void DocumentHandle_RejectsAliasTypesWithDifferentSchemaIdentity() - { - var handle = CultMesh.Document( - "mesh.note.current", - CultMesh.Verse("starbridge", "unity-pilot"), - _ => Task.FromResult(new MeshNoteDocument - { - Schema = "tests.mesh_note.v1", - Text = "primary", - Revision = 1 - }), - _ => new Subject()); - - Action act = () => handle.AsSchemaAlias(); - - act.Should() - .Throw() - .WithMessage("*tests.mesh_other*tests.mesh_note*"); - } - - [Test] - public async Task DocumentCatalog_IndexesHandlesByTypeAndSchemaAlias() + public async Task DocumentCatalog_IndexesHandlesByTypeAndSchema() { var current = new MeshNoteDocument { @@ -836,15 +806,12 @@ public sealed class CultMeshStreamingTests .Should() .BeSameAs(handle); - catalog.TryGetDocument(out var alias).Should().BeTrue(); - alias.DocumentId.Should().Be(handle.DocumentId); - alias.SchemaName.Should().Be(handle.SchemaName); - alias.SchemaVersion.Should().Be(handle.SchemaVersion); + catalog.TryGetDocument(out _).Should().BeFalse(); - var aliasSnapshot = await catalog.LatestAsync(); - aliasSnapshot.Text.Should().Be("catalog-primary"); - aliasSnapshot.Revision.Should().Be(3); - catalog.Latest().Text.Should().Be("catalog-primary"); + var snapshot = await catalog.LatestAsync(); + snapshot.Text.Should().Be("catalog-primary"); + snapshot.Revision.Should().Be(3); + catalog.Latest().Text.Should().Be("catalog-primary"); } [Test] @@ -894,21 +861,18 @@ public sealed class CultMeshStreamingTests cache, key, CultMesh.Verse("starbridge", "unity-pilot")); - var alias = handle.AsSchemaAlias(); handle.CanReplace.Should().BeTrue(); - alias.CanReplace.Should().BeTrue(); handle.CanSubmitPrediction.Should().BeFalse(); - alias.CanSubmitPrediction.Should().BeFalse(); handle.DocumentId.Should().Be(key.Value); handle.Sources.Should().ContainSingle().Which.SchemaId.Should().Be(handle.SchemaId); - var snapshot = await alias.LatestAsync(); + var snapshot = await handle.LatestAsync(); snapshot.Text.Should().Be("cache-initial"); - MeshNoteAliasDocument observed = null!; - using var subscription = alias.Watch(value => observed = value); - await alias.ReplaceAsync(new MeshNoteAliasDocument + MeshNoteDocument observed = null!; + using var subscription = handle.Watch(value => observed = value); + await handle.ReplaceAsync(new MeshNoteDocument { Schema = "tests.mesh_note.v1", Text = "cache-replaced", @@ -920,8 +884,8 @@ public sealed class CultMeshStreamingTests observed.Revision.Should().Be(2); var catalog = CultMesh.Documents(handle); - catalog.CanReplace().Should().BeTrue(); - await catalog.ReplaceAsync(new MeshNoteAliasDocument + catalog.CanReplace().Should().BeTrue(); + await catalog.ReplaceAsync(new MeshNoteDocument { Schema = "tests.mesh_note.v1", Text = "cache-catalog-replaced", @@ -929,7 +893,7 @@ public sealed class CultMeshStreamingTests }); cache.Get(key)!.Text.Should().Be("cache-catalog-replaced"); - Func act = () => alias.SubmitPredictionAsync(new MeshNoteAliasDocument + Func act = () => handle.SubmitPredictionAsync(new MeshNoteDocument { Schema = "tests.mesh_note.v1", Text = "cache-predicted", @@ -938,11 +902,11 @@ public sealed class CultMeshStreamingTests await act.Should().ThrowAsync(); } - // A database serves only what its shards own, including when a handle reads a same-schema instance of another CLR type - // out of the cache behind it. + // A database serves only what its shards own. A schema has one CLR type (retire-schema-alias), so there is no second type + // that could reach a record around the shard that owns its schema. [TestCase("tests.mesh_note.v1", true)] [TestCase("tests.unrelated.v1", false)] - public async Task DocumentHandle_ReadsASameSchemaInstanceOnlyWhenAShardOwnsIt(string ownedSchemaId, bool served) + public async Task DocumentHandle_ReadsARecordOnlyWhenAShardOwnsItsSchema(string ownedSchemaId, bool served) { var cache = new CultCache(); var key = new CultRecordKey("note:one"); @@ -952,7 +916,7 @@ public sealed class CultMeshStreamingTests RuntimeId = "reader", Shards = new[] { new CultNetShardDescriptor("notes", "reader", epoch: 1, isPrimary: true, schemaIds: new[] { ownedSchemaId }) } }); - var handle = CultMesh.Document(database, key, CultMesh.Verse("starbridge", "reader")); + var handle = CultMesh.Document(database, key, CultMesh.Verse("starbridge", "reader")); if (served) { @@ -996,19 +960,17 @@ public sealed class CultMeshStreamingTests database, key, CultMesh.Verse("starbridge", "pilot-a")); - var alias = handle.AsSchemaAlias(); handle.CanReplace.Should().BeFalse(); - alias.CanReplace.Should().BeFalse(); - alias.CanSubmitPrediction.Should().BeTrue(); + handle.CanSubmitPrediction.Should().BeTrue(); var catalog = CultMesh.Documents(handle); - catalog.CanSubmitPrediction().Should().BeTrue(); - var predictionWriter = catalog.PredictionWriter(); + catalog.CanSubmitPrediction().Should().BeTrue(); + var predictionWriter = catalog.PredictionWriter(); predictionWriter.Kind.Should().Be(CultMeshDocumentWriteKind.Prediction); MeshNoteDocument observed = null!; using var subscription = handle.Watch(value => observed = value); - await predictionWriter.WriteAsync(new MeshNoteAliasDocument + await predictionWriter.WriteAsync(new MeshNoteDocument { Schema = "tests.mesh_note.v1", Text = "predicted-thermal", @@ -1020,7 +982,7 @@ public sealed class CultMeshStreamingTests observed.Revision.Should().Be(5); (await handle.LatestAsync()).Text.Should().Be("predicted-thermal"); - var updated = await alias.PredictionWriter().UpdateAsync(value => new MeshNoteAliasDocument + var updated = await handle.PredictionWriter().UpdateAsync(value => new MeshNoteDocument { Schema = value.Schema, Text = "updated-as-prediction", @@ -1061,7 +1023,7 @@ public sealed class CultMeshStreamingTests }); var catalog = CultMesh.Documents(handle); - using var reactive = await catalog.PredictionWriter().ReactiveAsync( + using var reactive = await catalog.PredictionWriter().ReactiveAsync( new CultMeshReactiveDocumentOptions { FlushDelay = TimeSpan.FromMinutes(1) @@ -1671,7 +1633,7 @@ public sealed class CultMeshStreamingTests } [Test] - public async Task ReactiveDocument_SyncsSameSchemaAliasesAcrossRuntimeHandles() + public async Task ReactiveDocument_SyncsAcrossRuntimeHandles() { var cache = new CultCache(); var schemaId = CultDocumentRegistry.Shared.GetRequired().SchemaId; @@ -1712,13 +1674,11 @@ public sealed class CultMeshStreamingTests database, key, CultMesh.Verse("starbridge", "commander-rts")); - var observedByCommander = new List(); - using var commanderSubscription = commanderHandle - .AsSchemaAlias() - .Watch(observedByCommander.Add); + var observedByCommander = new List(); + using var commanderSubscription = commanderHandle.Watch(observedByCommander.Add); using var pilotReactive = await CultMesh .Documents(pilotHandle) - .PredictionWriter() + .PredictionWriter() .ReactiveAsync( new CultMeshReactiveDocumentOptions { @@ -1736,7 +1696,7 @@ public sealed class CultMeshStreamingTests commanderHandle.Context.RuntimeId.Should().Be("commander-rts"); observedByCommander.Last().Revision.Should().Be(2); - (await commanderHandle.AsSchemaAlias().LatestAsync()).Text.Should().Be("pilot-predicted"); + (await commanderHandle.LatestAsync()).Text.Should().Be("pilot-predicted"); } [Test] @@ -1781,12 +1741,9 @@ public sealed class CultMeshStreamingTests database, key, CultMesh.Verse("starbridge", "commander-rts")); - MeshNoteAliasDocument commanderSnapshot = null!; - using var commanderSubscription = commanderHandle - .AsSchemaAlias() - .Watch(value => commanderSnapshot = value); + MeshNoteDocument commanderSnapshot = null!; + using var commanderSubscription = commanderHandle.Watch(value => commanderSnapshot = value); using var pilotReactive = await pilotHandle - .AsSchemaAlias() .PredictionWriter() .ReactiveAsync(new CultMeshReactiveDocumentOptions { @@ -1814,7 +1771,7 @@ public sealed class CultMeshStreamingTests } [Test] - public async Task NodeDocument_OpensExplicitPredictionWriterForSameSchemaAlias() + public async Task NodeDocument_OpensExplicitPredictionWriter() { var schemaId = CultDocumentRegistry.Shared.GetRequired().SchemaId; using var temp = new TemporaryDirectory(); @@ -1854,7 +1811,7 @@ public sealed class CultMeshStreamingTests }, new CultRecordHandle(key)); using var pilotReactive = await node - .Document(key, CultMesh.Verse("starbridge", "pilot-a")) + .Document(key, CultMesh.Verse("starbridge", "pilot-a")) .PredictionWriter() .ReactiveAsync(new CultMeshReactiveDocumentOptions { @@ -1877,7 +1834,7 @@ public sealed class CultMeshStreamingTests pilotReactive.Document.DocumentId.Should().Be(key.Value); pilotReactive.Document.Context.RuntimeId.Should().Be("pilot-a"); node.Cache.Get(key)!.Text.Should().Be("node-one-call-prediction"); - (await commanderHandle.AsSchemaAlias().LatestAsync()).Revision.Should().Be(2); + (await commanderHandle.LatestAsync()).Revision.Should().Be(2); } [Test] @@ -1923,8 +1880,8 @@ public sealed class CultMeshStreamingTests DocumentRegistry = registry } }); - var handle = surface.SyncTo(node).Document(key.Value); - MeshNoteAliasDocument observed = null!; + var handle = surface.SyncTo(node).Document(key.Value); + MeshNoteDocument observed = null!; using var subscription = handle.Watch(value => observed = value); await sourceCache.UpsertAsync(new MeshNoteDocument { @@ -1982,7 +1939,7 @@ public sealed class CultMeshStreamingTests } })) { - var stale = await surface.SyncTo(crashedRuntime).Document(key.Value).LatestAsync(); + var stale = await surface.SyncTo(crashedRuntime).Document(key.Value).LatestAsync(); stale.Text.Should().Be("before-crash"); } @@ -2009,7 +1966,7 @@ public sealed class CultMeshStreamingTests var reconstructed = await surface .SyncTo(freshRuntime) .Documents(CultMesh.SnapshotDocument(key.Value)) - .LatestAsync(); + .LatestAsync(); reconstructed.Text.Should().Be("after-reboot-authority"); reconstructed.Revision.Should().Be(4); @@ -2060,7 +2017,7 @@ public sealed class CultMeshStreamingTests }); var readOnlyHandle = surface .SyncTo(node) - .Document(key.Value); + .Document(key.Value); readOnlyHandle.CanReplace.Should().BeFalse(); readOnlyHandle.CanSubmitPrediction.Should().BeFalse(); @@ -2628,34 +2585,6 @@ public sealed class CultMeshStreamingTests cache.Get(key)!.Text.Should().Be("published-directly"); } - [Test] - public void SingleFileDocumentHelpers_ReadLegacySingleDocumentSnapshots() - { - using var directory = new TemporaryDirectory(); - var filePath = Path.Combine(directory.Path, "legacy-single-document.ccmp"); - var key = new CultRecordKey("mesh-note:legacy-single-document"); - var descriptor = CultDocumentRegistry.Shared.GetRequired(); - var payload = CultDocumentMessagePackSerialization.SerializeUntyped(new MeshPublicationNoteDocument - { - Schema = "tests.mesh_publication_note.v1", - Text = "legacy-published", - Revision = 4 - }, typeof(MeshPublicationNoteDocument)); - - File.WriteAllBytes(filePath, WriteLegacySingleDocumentSnapshot( - key.Value, - descriptor.SchemaId, - descriptor.SchemaName, - descriptor.SchemaVersion, - "2026-06-27T12:00:00.0000000Z", - payload)); - - var document = CultMesh.ReadSingleFileDocument(filePath, key); - - document.Text.Should().Be("legacy-published"); - document.Revision.Should().Be(4); - } - [Test] public async Task DocumentHandle_ReadsRemotePeerSnapshotsAsTypedDocuments() { @@ -2671,7 +2600,7 @@ public sealed class CultMeshStreamingTests .Register(CultNetDocumentBinding.ForDocument(CultDocumentRegistry.Shared)); var requests = new List(); - var handle = CultMesh.DocumentFromPublication( + var handle = CultMesh.DocumentFromPublication( CultMeshDocumentPublicationSource.PeerSnapshot( () => new MeshSnapshotSchemaClient(request => { @@ -2692,16 +2621,16 @@ public sealed class CultMeshStreamingTests snapshot.Text.Should().Be("remote-snapshot"); snapshot.Revision.Should().Be(9); - handle.DocumentType.Should().Be(typeof(MeshNoteAliasDocument)); + handle.DocumentType.Should().Be(typeof(MeshNoteDocument)); handle.RouteHint.Kind.Should().Be(CultMeshLocalityKind.Network); handle.Sources.Should().ContainSingle().Which.SourceId.Should().Be(key.Value); requests.Should().ContainSingle(); requests[0].SchemaIds.Should().ContainSingle().Which .Should().Be(CultDocumentRegistry.Shared.GetRequired().SchemaId); requests[0].RecordKeys.Should().ContainSingle().Which.Should().Be(key.Value); - (await catalog.LatestAsync()).Text.Should().Be("remote-snapshot"); + (await catalog.LatestAsync()).Text.Should().Be("remote-snapshot"); - var aliasSchemaHandle = CultMesh.DocumentFromPeerSnapshot( + var aliasSchemaHandle = CultMesh.DocumentFromPeerSnapshot( _ => Task.FromResult(new CultNetSnapshotResponseRawMessage { MessageId = "alias-schema", @@ -2729,7 +2658,7 @@ public sealed class CultMeshStreamingTests (await aliasSchemaHandle.LatestAsync()).Text.Should().Be("record-key-fallback"); - var mixedAliasSchemaHandle = CultMesh.DocumentFromPeerSnapshot( + var mixedAliasSchemaHandle = CultMesh.DocumentFromPeerSnapshot( _ => Task.FromResult(new CultNetSnapshotResponseRawMessage { MessageId = "alias-schema-with-neighbor", @@ -2775,7 +2704,7 @@ public sealed class CultMeshStreamingTests // The recordKey match is exact (ordinal), not case-insensitive. A case-variant key must not // win even when it appears first in the response. var caseVariantKey = key.Value.ToUpperInvariant(); - var caseVariantKeyHandle = CultMesh.DocumentFromPeerSnapshot( + var caseVariantKeyHandle = CultMesh.DocumentFromPeerSnapshot( _ => Task.FromResult(new CultNetSnapshotResponseRawMessage { MessageId = "case-variant-key", @@ -2888,7 +2817,7 @@ public sealed class CultMeshStreamingTests CultMesh.PublicationDocument( firstKey, documentId: "daemon:first"), - CultMesh.PublicationDocument( + CultMesh.PublicationDocument( secondKey, documentId: "daemon:second", source: CultMeshDocumentPublicationSource.SingleFile(secondPath)) @@ -2902,10 +2831,10 @@ public sealed class CultMeshStreamingTests catalog.Documents.Select(document => document.DocumentId).Should().Equal("daemon:first", "daemon:second"); (await catalog.LatestAsync()).Text.Should().Be("first source"); - (await catalog.LatestAsync()).Text.Should().Be("second source"); + (await catalog.LatestAsync()).Text.Should().Be("second source"); catalog.Document().DocumentId.Should().Be("daemon:second"); catalog.Document().RouteHint.Description.Should().Be("publication catalog"); - catalog.Document().Sources.Should().ContainSingle().Which.SourceId.Should().Be("daemon:second"); + catalog.Document().Sources.Should().ContainSingle().Which.SourceId.Should().Be("daemon:second"); } [Test] @@ -2949,7 +2878,7 @@ public sealed class CultMeshStreamingTests CultMesh.PublicationDocument( firstKey, documentId: "local:first"), - CultMesh.PublicationDocument( + CultMesh.PublicationDocument( secondKey, documentId: "local:second", source: CultMeshDocumentPublicationSource.SingleFile(secondPath)) @@ -2962,16 +2891,16 @@ public sealed class CultMeshStreamingTests catalog.Documents.Select(document => document.DocumentId).Should().Equal("local:first", "local:second"); (await catalog.LatestAsync()).Text.Should().Be("first synced source"); - (await catalog.LatestAsync()).Text.Should().Be("second synced source"); + (await catalog.LatestAsync()).Text.Should().Be("second synced source"); node.Cache.Get(firstKey)!.Revision.Should().Be(31); (await node.Document(secondKey, CultMesh.Verse("starbridge", "unity-sync-catalog")) .LatestAsync()).Text.Should().Be("second synced source"); - (await node.Document(secondKey, CultMesh.Verse("starbridge", "unity-sync-catalog")) + (await node.Document(secondKey, CultMesh.Verse("starbridge", "unity-sync-catalog")) .LatestAsync()).Revision.Should().Be(32); } [Test] - public async Task PublicationHelpers_SyncConfiguredPeerSourceIntoLocalNodeAlias() + public async Task PublicationHelpers_SyncConfiguredPeerSourceIntoLocalNode() { var sourceCache = new CultCache(); var registry = new CultNetDocumentRegistry(CultDocumentRegistry.Shared); @@ -3000,7 +2929,7 @@ public sealed class CultMeshStreamingTests } }); - var synced = await CultMesh.SyncDocumentFromPublicationAsync( + var synced = await CultMesh.SyncDocumentFromPublicationAsync( node, CultMeshDocumentPublicationSource.PeerSnapshot( () => new MeshSnapshotSchemaClient(request => @@ -3016,12 +2945,12 @@ public sealed class CultMeshStreamingTests MessageIdPrefix = "mesh-test-publication-sync", PollInterval = TimeSpan.FromMilliseconds(10) }); - var localAlias = await node.Document(key, verse).LatestAsync(); + var local = await node.Document(key, verse).LatestAsync(); synced.Text.Should().Be("publication-synced"); synced.Revision.Should().Be(51); - localAlias.Text.Should().Be("publication-synced"); - localAlias.Revision.Should().Be(51); + local.Text.Should().Be("publication-synced"); + local.Revision.Should().Be(51); requests.Should().ContainSingle().Which.RecordKeys.Should().Equal(key.Value); } @@ -3128,7 +3057,7 @@ public sealed class CultMeshStreamingTests } [Test] - public async Task SnapshotHelpers_DecodeSameSchemaAliasFromForeignSchemaId() + public async Task SnapshotHelpers_DecodeFromForeignSchemaId() { var key = new CultRecordKey("mesh-note:snapshot-alias-foreign-schema"); var snapshot = new CultNetSnapshotResponseRawMessage @@ -3154,7 +3083,7 @@ public sealed class CultMeshStreamingTests } }; - var documents = await CultMesh.FetchSnapshotDocumentsAsync( + var documents = await CultMesh.FetchSnapshotDocumentsAsync( "cultnet://foreign-schema.test:3075", new CultMeshSnapshotRequestOptions { @@ -3198,9 +3127,9 @@ public sealed class CultMeshStreamingTests }; var cacheRegistry = CultMesh.CreateCultCacheDocumentRegistry( typeof(MeshNoteDocument), - typeof(MeshNoteAliasDocument)); + typeof(MeshNoteDocument)); var registry = CultMesh.CreateCultNetDocumentRegistry( - new[] { typeof(MeshNoteDocument), typeof(MeshNoteAliasDocument) }, + new[] { typeof(MeshNoteDocument), typeof(MeshNoteDocument) }, cacheRegistry); var endpoint = CultMesh.SnapshotEndpoint( "cultnet://foreign-schema.test:3075", @@ -3231,7 +3160,7 @@ public sealed class CultMeshStreamingTests } }); - var synced = await endpoint.SyncDocumentAsync(node, key.Value); + var synced = await endpoint.SyncDocumentAsync(node, key.Value); synced.Text.Should().Be("foreign-schema-synced"); node.Cache.Get(key)!.Text.Should().Be("foreign-schema-synced"); @@ -3239,7 +3168,7 @@ public sealed class CultMeshStreamingTests } [Test] - public async Task SnapshotFacade_SyncsRemoteDocumentIntoLocalNodeWithSameSchemaAlias() + public async Task SnapshotFacade_SyncsRemoteDocumentIntoLocalNode() { var sourceCache = new CultCache(); var registry = new CultNetDocumentRegistry(CultDocumentRegistry.Shared); @@ -3267,7 +3196,7 @@ public sealed class CultMeshStreamingTests } }); - var synced = await CultMesh.SyncDocumentFromPeerSnapshotAsync( + var synced = await CultMesh.SyncDocumentFromPeerSnapshotAsync( node, () => new MeshSnapshotSchemaClient(request => { @@ -3281,12 +3210,12 @@ public sealed class CultMeshStreamingTests DocumentRegistry = registry, Context = CultMesh.Verse("starbridge", "unity-raven").Context }); - var localAlias = await node.Document(key.Value, CultMesh.Verse("starbridge", "unity-raven")) + var local = await node.Document(key.Value, CultMesh.Verse("starbridge", "unity-raven")) .LatestAsync(); synced.Text.Should().Be("facade-synced"); synced.Revision.Should().Be(46); - localAlias.Text.Should().Be("facade-synced"); + local.Text.Should().Be("facade-synced"); node.Cache.Get(key)!.Revision.Should().Be(46); requests.Should().ContainSingle().Which.RecordKeys.Should().Equal(key.Value); } @@ -3326,9 +3255,9 @@ public sealed class CultMeshStreamingTests var sourceCache = new CultCache(); var cacheRegistry = CultMesh.CreateCultCacheDocumentRegistry( typeof(MeshNoteDocument), - typeof(MeshNoteAliasDocument)); + typeof(MeshNoteDocument)); var networkRegistry = CultMesh.CreateCultNetDocumentRegistry( - new[] { typeof(MeshNoteDocument), typeof(MeshNoteAliasDocument) }, + new[] { typeof(MeshNoteDocument), typeof(MeshNoteDocument) }, cacheRegistry); var key = new CultRecordKey("mesh-note:registry-helper"); await sourceCache.UpsertAsync(new MeshNoteDocument @@ -3339,7 +3268,7 @@ public sealed class CultMeshStreamingTests }, new CultRecordHandle(key)); var endpoint = "cultnet://registry-helper.test:3077"; - var documents = await CultMesh.FetchSnapshotDocumentsAsync( + var documents = await CultMesh.FetchSnapshotDocumentsAsync( endpoint, new CultMeshSnapshotRequestOptions { @@ -3350,15 +3279,15 @@ public sealed class CultMeshStreamingTests networkRegistry); var schemaId = cacheRegistry.GetRequired().SchemaId; - cacheRegistry.GetRequired().SchemaId.Should().Be(schemaId); + cacheRegistry.GetRequired().SchemaId.Should().Be(schemaId); + networkRegistry.GetByDocumentType(typeof(MeshNoteDocument)).Should().NotBeNull(); networkRegistry.GetByDocumentType(typeof(MeshNoteDocument)).Should().NotBeNull(); - networkRegistry.GetByDocumentType(typeof(MeshNoteAliasDocument)).Should().NotBeNull(); networkRegistry.GetBySchemaId(schemaId)!.DocumentType.Should().Be(typeof(MeshNoteDocument)); documents.Should().ContainSingle().Which.Text.Should().Be("registry-helper"); } [Test] - public async Task SnapshotEndpoint_ProvidesTypedHandlesAndSchemaAliases() + public async Task SnapshotEndpoint_ProvidesTypedHandles() { var sourceCache = new CultCache(); var registry = new CultNetDocumentRegistry(CultDocumentRegistry.Shared); @@ -3390,7 +3319,7 @@ public sealed class CultMeshStreamingTests } }); - var fetchedAlias = await surface.FetchDocumentAsync(key.Value); + var fetched = await surface.FetchDocumentAsync(key.Value); using var node = await CultMesh.CreateNodeAsync( Path.Combine(Path.GetTempPath(), $"cultmesh-snapshot-endpoint-{Guid.NewGuid():N}.ccmp"), new CultMeshNodeOptions @@ -3405,12 +3334,12 @@ public sealed class CultMeshStreamingTests DocumentRegistry = registry } }); - var syncedAlias = await surface.SyncDocumentAsync(node, key.Value); - var aliasHandle = surface.Document(key.Value); - var aliasLatest = await aliasHandle.LatestAsync(); + var syncedDocument = await surface.SyncDocumentAsync(node, key.Value); + var endpointHandle = surface.Document(key.Value); + var endpointLatest = await endpointHandle.LatestAsync(); var catalog = surface.Documents( CultMesh.SnapshotDocument(key.Value, "daemon:mesh-note:snapshot-endpoint")); - var catalogAlias = await catalog.LatestAsync(); + var catalogLatest = await catalog.LatestAsync(); await sourceCache.UpsertAsync(new MeshNoteDocument { Schema = "tests.mesh_note.v1", @@ -3418,7 +3347,7 @@ public sealed class CultMeshStreamingTests Revision = 20 }, new CultRecordHandle(key)); var syncedEndpoint = surface.SyncTo(node); - var syncedHandle = syncedEndpoint.Document(key.Value); + var syncedHandle = syncedEndpoint.Document(key.Value); var syncedLatest = await syncedHandle.LatestAsync(); await sourceCache.UpsertAsync(new MeshNoteDocument { @@ -3428,16 +3357,16 @@ public sealed class CultMeshStreamingTests }, new CultRecordHandle(key)); var syncedCatalog = syncedEndpoint.Documents( CultMesh.SnapshotDocument(key.Value)); - var syncedCatalogAlias = await syncedCatalog.LatestAsync(); + var syncedCatalogLatest = await syncedCatalog.LatestAsync(); - fetchedAlias.Text.Should().Be("snapshot-endpoint"); - syncedAlias.Text.Should().Be("snapshot-endpoint"); - aliasLatest.Revision.Should().Be(19); + fetched.Text.Should().Be("snapshot-endpoint"); + syncedDocument.Text.Should().Be("snapshot-endpoint"); + endpointLatest.Revision.Should().Be(19); catalog.Document().DocumentId.Should().Be("daemon:mesh-note:snapshot-endpoint"); - catalogAlias.Text.Should().Be("snapshot-endpoint"); + catalogLatest.Text.Should().Be("snapshot-endpoint"); syncedLatest.Text.Should().Be("snapshot-endpoint-synced"); node.Cache.Get(key)!.Revision.Should().Be(21); - syncedCatalogAlias.Text.Should().Be("snapshot-endpoint-catalog-synced"); + syncedCatalogLatest.Text.Should().Be("snapshot-endpoint-catalog-synced"); requests.Should().HaveCount(6); requests.Should().OnlyContain(request => request.SchemaIds == null && @@ -3490,7 +3419,7 @@ public sealed class CultMeshStreamingTests } [Test] - public async Task CollectionHandle_SupportsSameSchemaAliases() + public async Task CollectionHandle_IsIndexedInItsCatalogByTypeAndSchema() { var cache = new CultCache(); await cache.UpsertAsync( @@ -3503,14 +3432,12 @@ public sealed class CultMeshStreamingTests new CultRecordHandle(new CultRecordKey("mesh-note:collection"))); var collection = CultMesh.Collection(cache); - var alias = collection.AsSchemaAlias(); var catalog = CultMesh.Collections(collection); - alias.CollectionId.Should().Be(collection.CollectionId); - alias.SchemaName.Should().Be(collection.SchemaName); catalog.Collections.Should().ContainSingle().Which.Should().BeSameAs(collection); - catalog.TryGetCollection(out var catalogAlias).Should().BeTrue(); - catalogAlias.CollectionId.Should().Be(collection.CollectionId); + catalog.TryGetCollection(out var byType).Should().BeTrue(); + byType.Should().BeSameAs(collection); + catalog.TryGetCollection(out _).Should().BeFalse(); catalog.TryGetCollectionBySchema("tests.mesh_note.v1", out var byVersion).Should().BeTrue(); byVersion.Should().BeSameAs(collection); catalog.CollectionBySchema("tests.mesh_note").Should().BeSameAs(collection); @@ -3518,13 +3445,13 @@ public sealed class CultMeshStreamingTests .Should() .BeSameAs(collection); - var snapshot = await alias.LatestAsync(); + var snapshot = await collection.LatestAsync(); snapshot.Should().ContainSingle().Which.Text.Should().Be("collection-alias"); - var catalogSnapshot = await catalog.LatestAsync(); + var catalogSnapshot = await catalog.LatestAsync(); catalogSnapshot.Should().ContainSingle().Which.Revision.Should().Be(4); - CultMeshCollectionChange observed = null!; - using var subscription = catalog.WatchChanges(change => observed = change); + CultMeshCollectionChange observed = null!; + using var subscription = catalog.WatchChanges(change => observed = change); await cache.UpsertAsync( new MeshNoteDocument { @@ -3569,11 +3496,10 @@ public sealed class CultMeshStreamingTests node, key, CultMesh.Verse("starbridge", "unity-pilot")); - var alias = handle.AsSchemaAlias(); MeshNoteDocument observed = null!; using var subscription = handle.Watch(value => observed = value); - await alias.ReplaceAsync(new MeshNoteAliasDocument + await handle.ReplaceAsync(new MeshNoteDocument { Schema = "tests.mesh_note.v1", Text = "node-replaced", @@ -3899,39 +3825,6 @@ public sealed class CultMeshStreamingTests ring.Stats().UnavoidableCopyCount.Should().Be(1); } - private static byte[] WriteLegacySingleDocumentSnapshot( - string key, - string schemaId, - string schemaName, - string schemaVersion, - string storedAt, - byte[] payload) - { - var buffer = new ArrayBufferWriter(); - var writer = new MessagePackWriter(buffer); - - writer.WriteArrayHeader(3); - writer.Write("cultcache.store.v1"); - writer.WriteArrayHeader(1); - writer.WriteArrayHeader(7); - writer.Write(schemaId); - writer.Write(schemaName); - writer.Write(schemaVersion); - writer.Write("Legacy Aetheria single document"); - writer.Write(storedAt); - writer.Write("legacy-hash"); - writer.WriteArrayHeader(0); - writer.WriteArrayHeader(1); - writer.WriteArrayHeader(4); - writer.Write(key); - writer.Write(schemaId); - writer.Write(storedAt); - writer.Write(payload); - writer.Flush(); - - return buffer.WrittenSpan.ToArray(); - } - private static CultMeshStreamCatalog CatalogWithByteStream() { var catalog = CultMesh.CreateStreamCatalog(); @@ -4011,20 +3904,6 @@ public sealed class CultMeshStreamingTests public int Revision { get; set; } } - [CultDocument("tests.mesh_note", "tests.mesh_note.v1")] - [MessagePackObject(AllowPrivate = true)] - internal sealed class MeshNoteAliasDocument - { - [Key(0)] - public string Schema { get; set; } = string.Empty; - - [Key(1)] - public string Text { get; set; } = string.Empty; - - [Key(2)] - public int Revision { get; set; } - } - [CultDocument("tests.mesh_publication_note", "tests.mesh_publication_note.v1")] [MessagePackObject(AllowPrivate = true)] internal sealed class MeshPublicationNoteDocument diff --git a/tests/GameCult.Mesh.Tests/CultMeshVariantRefusalTests.cs b/tests/GameCult.Mesh.Tests/CultMeshVariantRefusalTests.cs index 2521aa8a..94f05d34 100644 --- a/tests/GameCult.Mesh.Tests/CultMeshVariantRefusalTests.cs +++ b/tests/GameCult.Mesh.Tests/CultMeshVariantRefusalTests.cs @@ -2,6 +2,7 @@ using System; using System.IO; using FluentAssertions; using GameCult.Caching; +using GameCult.Caching.MessagePack; using NUnit.Framework; #nullable enable @@ -23,8 +24,8 @@ public sealed class CultMeshVariantRefusalTests throw new FileNotFoundException($"Shared vector {name} not found above {TestContext.CurrentContext.TestDirectory}."); } - private static NotSupportedException Refusal(string vector) => - Assert.Throws(() => + private static CultStoreUnreadableException Refusal(string vector) => + Assert.Throws(() => CultMesh.ReadSingleFileDocumentPayload(VectorPath(vector), new CultRecordKey("item:bellows"), ItemSchemaId))!; [Test] @@ -38,11 +39,47 @@ public sealed class CultMeshVariantRefusalTests message.Should().Contain("item:anvil").And.Contain(ItemSchemaId); } - [Test] - public void ReadSingleFileDocument_RefusesExtraRecordSlotInTheLegacyCatalogLayoutToo() + // The older catalog layout (the content hash at entry slot 5) is a store no C# reader reads: CultMesh refuses it as CultCache + // does, on a read and on a raw write, and a refused write leaves the bytes as they were. + [TestCase("legacy-catalog-plain.msgpack")] + [TestCase("legacy-catalog-extra-slot.msgpack")] + [TestCase("legacy-catalog-trailing.msgpack")] + public void TheOlderCatalogLayoutIsRefusedByMeshAndCultCacheAlike(string vector) { - var message = Refusal("legacy-catalog-extra-slot.msgpack").Message; - message.Should().Contain("item:anvil").And.Contain(ItemSchemaId); + Refusal(vector).Path.Should().Be(VectorPath(vector)); + + var directory = Path.Combine(Path.GetTempPath(), "cultmesh-legacy-layout", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(directory); + try + { + var path = Path.Combine(directory, vector); + File.Copy(VectorPath(vector), path); + var bytes = File.ReadAllBytes(path); + var raw = new CultMeshSingleFileDocumentSchema("raw:schema", "RawSchema", "1"); + + Assert.Throws(() => + CultMesh.WriteSingleFileDocumentPayload(path, new CultRecordKey("item:bellows"), raw, null, new byte[] { 0x90 }))! + .Path.Should().Be(path); + File.ReadAllBytes(path).Should().Equal(bytes); + + Assert.Throws(() => CultCacheMessagePack.Create(path, new CultCacheOpenOptions { FlushOnDispose = false, StoreFlushOnDispose = false }))! + .Path.Should().Be(path); + } + finally + { + Directory.Delete(directory, true); + } + } + + // A store in the current layout with one catalog slot malformed is refused as CultCache refuses it. + [TestCase("current-catalog-bad-canonical.msgpack")] + [TestCase("current-catalog-bad-members.msgpack")] + [TestCase("current-catalog-bad-compat.msgpack")] + [TestCase("current-catalog-bad-slot5-nil.msgpack")] + [TestCase("current-catalog-bad-slot5-int.msgpack")] + public void ReadSingleFileDocument_RefusesAMalformedCurrentCatalogEntry(string vector) + { + Refusal(vector).Path.Should().Be(VectorPath(vector)); } // Q6: CultMesh does not resolve variants. It refuses a variant key naming it and its base, and still reads the plain diff --git a/tests/GameCult.Networking.Tests/CultNetDocumentRegistryConflictTests.cs b/tests/GameCult.Networking.Tests/CultNetDocumentRegistryConflictTests.cs new file mode 100644 index 00000000..6f48ee7e --- /dev/null +++ b/tests/GameCult.Networking.Tests/CultNetDocumentRegistryConflictTests.cs @@ -0,0 +1,91 @@ +#nullable enable +using System; +using GameCult.Caching; +using MessagePack; +using NUnit.Framework; + +namespace GameCult.Networking.Tests +{ + // F7 (operator, 2026-09-30): "Schema aliasing sounds like a really bad idea, I support retiring it." One binding owns a + // document type and a schema id, in the reference runtime as in Rust, Kotlin and TypeScript. + public sealed class CultNetDocumentRegistryConflictTests + { + private static CultNetDocumentBinding Bind(string? schemaId = null) where T : class => + CultNetDocumentBinding.ForDocument(CultDocumentRegistry.Shared, schemaId); + + // Two bindings stand ahead of the holder, so a conflict check that looks only at the first holder misses it. + private static CultNetDocumentRegistry RegistryHolding(CultNetDocumentBinding holder) => + new CultNetDocumentRegistry(CultDocumentRegistry.Shared) + .Register(Bind()) + .Register(Bind()) + .Register(holder); + + [Test] + public void A_second_type_for_one_schema_id_is_refused_typed_in_either_order() + { + var first = Bind("f5.shared"); + var second = Bind("f5.shared"); + foreach (var (held, claimant) in new[] { (first, second), (second, first) }) + { + var registry = RegistryHolding(held); + var error = Assert.Throws(() => registry.Register(claimant))!; + Assert.That(error.SchemaId, Is.EqualTo("f5.shared")); + Assert.That(error.SchemaNames, Is.EqualTo(new[] { held.DocumentType.FullName, claimant.DocumentType.FullName })); + Assert.That(error.RecordKey, Is.EqualTo(string.Empty)); + Assert.That(error.Message, Does.Contain($"type \"{claimant.DocumentType.FullName}\"")); + // Nothing was bound: the schema id still resolves to the first type, and the claimant's type to nothing. + Assert.That(registry.GetBySchemaId("f5.shared"), Is.SameAs(held)); + Assert.That(registry.GetByDocumentType(claimant.DocumentType), Is.Null); + } + } + + [Test] + public void A_type_bound_to_one_schema_id_cannot_be_bound_to_another() + { + var own = Bind(); + var registry = RegistryHolding(own); + var error = Assert.Throws(() => registry.Register(Bind("f5.elsewhere")))!; + Assert.That(error.SchemaId, Is.EqualTo("f5.elsewhere")); + Assert.That(registry.GetBySchemaId("f5.elsewhere"), Is.Null); + Assert.That(registry.GetByDocumentType(typeof(ConflictNote)), Is.SameAs(own)); + } + + [Test] + public void Binding_a_type_to_the_schema_id_it_holds_again_replaces_the_binding() + { + var registry = RegistryHolding(Bind()); + var again = Bind(); + registry.Register(again); + Assert.That(registry.GetByDocumentType(typeof(ConflictNote)), Is.SameAs(again)); + Assert.That(registry.GetBySchemaId(again.SchemaId), Is.SameAs(again)); + } + + [CultDocument("f5.conflict-note", "f5.conflict-note.v1")] + [MessagePackObject(AllowPrivate = true)] + public sealed class ConflictNote + { + [Key(0)] public string Body = string.Empty; + } + + [CultDocument("f5.conflict-other-note", "f5.conflict-other-note.v1")] + [MessagePackObject(AllowPrivate = true)] + public sealed class ConflictOtherNote + { + [Key(0)] public string Body = string.Empty; + } + + [CultDocument("f5.conflict-filler-a", "f5.conflict-filler-a.v1")] + [MessagePackObject(AllowPrivate = true)] + public sealed class ConflictFillerA + { + [Key(0)] public string Body = string.Empty; + } + + [CultDocument("f5.conflict-filler-b", "f5.conflict-filler-b.v1")] + [MessagePackObject(AllowPrivate = true)] + public sealed class ConflictFillerB + { + [Key(0)] public string Body = string.Empty; + } + } +} diff --git a/tests/GameCult.Networking.Tests/CultNetSelectionFixBatchTests.cs b/tests/GameCult.Networking.Tests/CultNetSelectionFixBatchTests.cs index e4c31d64..9ecc4556 100644 --- a/tests/GameCult.Networking.Tests/CultNetSelectionFixBatchTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetSelectionFixBatchTests.cs @@ -600,6 +600,7 @@ namespace GameCult.Networking.Tests "selection.fixture.guard_probe", "selection.fixture.guard_probe.v1", "selection.fixture.guard_probe.v1", + Array.Empty(), "sha256:guard-probe", "{}", false, diff --git a/tests/vectors/document-variants-c0/current-catalog-bad-canonical.msgpack b/tests/vectors/document-variants-c0/current-catalog-bad-canonical.msgpack new file mode 100644 index 00000000..cd6bd37f --- /dev/null +++ b/tests/vectors/document-variants-c0/current-catalog-bad-canonical.msgpack @@ -0,0 +1,2 @@ +“²cultcache.store.v1‘•ÙGsha256:88d3fdf0a927acf3b163940d8f8c7fe62b3316542ce771a67ec8bc038f594788µfixtures.precut2.item¸fixtures.precut2.item.v1ÙGsha256:dadf30bc920849ee89ed5d70cc73027070dd7571c5f878db1901f3abc459cc0c’”ªitem:anvilÙGsha256:88d3fdf0a927acf3b163940d8f8c7fe62b3316542ce771a67ec8bc038f594788Ù!2026-09-13T15:14:36.4032226+00:00Ä’¥anvil”¬item:bellowsÙGsha256:88d3fdf0a927acf3b163940d8f8c7fe62b3316542ce771a67ec8bc038f594788Ù!2026-09-13T15:14:36.4114746+00:00Ä +’§bellows* \ No newline at end of file diff --git a/tests/vectors/document-variants-c0/current-catalog-bad-compat.msgpack b/tests/vectors/document-variants-c0/current-catalog-bad-compat.msgpack new file mode 100644 index 00000000..e00f8bf4 Binary files /dev/null and b/tests/vectors/document-variants-c0/current-catalog-bad-compat.msgpack differ diff --git a/tests/vectors/document-variants-c0/current-catalog-bad-members.msgpack b/tests/vectors/document-variants-c0/current-catalog-bad-members.msgpack new file mode 100644 index 00000000..4a900c8e --- /dev/null +++ b/tests/vectors/document-variants-c0/current-catalog-bad-members.msgpack @@ -0,0 +1,2 @@ +“²cultcache.store.v1‘—ÙGsha256:88d3fdf0a927acf3b163940d8f8c7fe62b3316542ce771a67ec8bc038f594788µfixtures.precut2.item¸fixtures.precut2.item.v1ÙGsha256:dadf30bc920849ee89ed5d70cc73027070dd7571c5f878db1901f3abc459cc0cÚp{"schemaName":"fixtures.precut2.item","schemaVersion":"fixtures.precut2.item.v1","members":[{"slot":0,"name":"Name","type":"System.String","isReference":false,"many":false,"targetSchemaName":null,"indexAlias":null,"isName":true},{"slot":1,"name":"Count","type":"System.Int32","isReference":false,"many":false,"targetSchemaName":null,"indexAlias":null,"isName":false}]}‘ÙGsha256:88d3fdf0a927acf3b163940d8f8c7fe62b3316542ce771a67ec8bc038f594788’”ªitem:anvilÙGsha256:88d3fdf0a927acf3b163940d8f8c7fe62b3316542ce771a67ec8bc038f594788Ù!2026-09-13T15:14:36.4032226+00:00Ä’¥anvil”¬item:bellowsÙGsha256:88d3fdf0a927acf3b163940d8f8c7fe62b3316542ce771a67ec8bc038f594788Ù!2026-09-13T15:14:36.4114746+00:00Ä +’§bellows* \ No newline at end of file diff --git a/tests/vectors/document-variants-c0/current-catalog-bad-slot5-int.msgpack b/tests/vectors/document-variants-c0/current-catalog-bad-slot5-int.msgpack new file mode 100644 index 00000000..5f21bf61 Binary files /dev/null and b/tests/vectors/document-variants-c0/current-catalog-bad-slot5-int.msgpack differ diff --git a/tests/vectors/document-variants-c0/current-catalog-bad-slot5-nil.msgpack b/tests/vectors/document-variants-c0/current-catalog-bad-slot5-nil.msgpack new file mode 100644 index 00000000..f6fc971f Binary files /dev/null and b/tests/vectors/document-variants-c0/current-catalog-bad-slot5-nil.msgpack differ diff --git a/tests/vectors/document-variants-c0/generate.py b/tests/vectors/document-variants-c0/generate.py index c6e5d2da..75326dac 100644 --- a/tests/vectors/document-variants-c0/generate.py +++ b/tests/vectors/document-variants-c0/generate.py @@ -6,8 +6,20 @@ at test time. Run: python generate.py (needs msgpack). extra-slot-full-payload.msgpack v1 header, record item:anvil gains slot 4, payload stays full legacy-catalog-extra-slot.msgpack v1 header, catalog entry in the older layout (contentHash at slot - 5), so CultMesh's legacy snapshot reader takes it, plus the - extra-slot record + 5), plus the extra-slot record; no C# reader reads the older layout + legacy-catalog-plain.msgpack v1 header, the older-layout catalog entry, the plain v1 records: refused by + every C# reader, CultMesh included + legacy-catalog-trailing.msgpack legacy-catalog-plain plus three bytes after the array + current-catalog-bad-canonical.msgpack + current-catalog-bad-members.msgpack + current-catalog-bad-compat.msgpack + v1 header, plain records, and the first catalog entry of the current layout with one slot + malformed (canonical schema json an int; members an int; a compatible id an int). Refused + by every C# reader, CultMesh included + current-catalog-bad-slot5-nil.msgpack + current-catalog-bad-slot5-int.msgpack + the same, with slot 5 (the compatible ids) nil or an integer: neither is a string, so neither is + the older layout v1-base.msgpack NOT generated here: written once by the Python runtime at CultLib 69a21bb (SingleFileMessagePackBackingStore.push of alpha and beta, type vectors.item, before any C0 code existed) @@ -38,6 +50,21 @@ write("extra-slot-full-payload.msgpack", [header, catalog, [anvil + [["item:bellows", []]]] + records[1:]]) variant = ["item:anvil-big", anvil[1], anvil[2], b"", ["item:anvil", [[0, [[1, ""]], "", 99]]]] write("variant-v2.msgpack", ["cultcache.store.v2", catalog, records + [variant]]) -legacy_entry = [catalog[0][0], catalog[0][1], catalog[0][2], catalog[0][4], None, catalog[0][3], []] +legacy = lambda entry: [entry[0], entry[1], entry[2], entry[4], None, entry[3], []] +legacy_entry = legacy(catalog[0]) write("legacy-catalog-extra-slot.msgpack", [header, [legacy_entry], [anvil + [["item:bellows", []]]]]) +write("legacy-catalog-plain.msgpack", [header, [legacy(entry) for entry in catalog], records]) +(here / "legacy-catalog-trailing.msgpack").write_bytes( + msgpack.packb([header, [legacy(entry) for entry in catalog], records], use_bin_type=True) + b"\x01\x02\x03") +current = catalog[0] +for name, entry in ( + ("current-catalog-bad-canonical", current[:4] + [5]), + ("current-catalog-bad-members", current[:6] + [5]), + ("current-catalog-bad-compat", current[:5] + [[7]] + current[6:]), + ("current-catalog-bad-slot5-nil", current[:5] + [None] + current[6:]), + ("current-catalog-bad-slot5-int", current[:5] + [7] + current[6:]), +): + # One catalog entry and the records that use it: the older-layout reader takes an entry's slot 5 for a string, so a second + # current-layout entry would make it refuse for that reason, not for the malformed slot under test. + write(name + ".msgpack", [header, [entry], [record for record in records if record[1] == current[0]]]) diff --git a/tests/vectors/document-variants-c0/legacy-catalog-plain.msgpack b/tests/vectors/document-variants-c0/legacy-catalog-plain.msgpack new file mode 100644 index 00000000..345d12f7 --- /dev/null +++ b/tests/vectors/document-variants-c0/legacy-catalog-plain.msgpack @@ -0,0 +1,2 @@ +“²cultcache.store.v1’—ÙGsha256:88d3fdf0a927acf3b163940d8f8c7fe62b3316542ce771a67ec8bc038f594788µfixtures.precut2.item¸fixtures.precut2.item.v1Úp{"schemaName":"fixtures.precut2.item","schemaVersion":"fixtures.precut2.item.v1","members":[{"slot":0,"name":"Name","type":"System.String","isReference":false,"many":false,"targetSchemaName":null,"indexAlias":null,"isName":true},{"slot":1,"name":"Count","type":"System.Int32","isReference":false,"many":false,"targetSchemaName":null,"indexAlias":null,"isName":false}]}ÀÙGsha256:dadf30bc920849ee89ed5d70cc73027070dd7571c5f878db1901f3abc459cc0c—ÙGsha256:670deacd06aedb90235e6bd873674c10467a05b4adcd62dbc019279efcbcbb11µfixtures.precut2.note¸fixtures.precut2.note.v1Ùç{"schemaName":"fixtures.precut2.note","schemaVersion":"fixtures.precut2.note.v1","members":[{"slot":0,"name":"Text","type":"System.String","isReference":false,"many":false,"targetSchemaName":null,"indexAlias":null,"isName":false}]}ÀÙGsha256:8552c4cd5a0a455fc43a37c2314d28617b7b9d36e43ee7ef2dd18cb148dbfe35“”ªitem:anvilÙGsha256:88d3fdf0a927acf3b163940d8f8c7fe62b3316542ce771a67ec8bc038f594788Ù!2026-09-13T15:14:36.4032226+00:00Ä’¥anvil”¬item:bellowsÙGsha256:88d3fdf0a927acf3b163940d8f8c7fe62b3316542ce771a67ec8bc038f594788Ù!2026-09-13T15:14:36.4114746+00:00Ä +’§bellows*”«note:originÙGsha256:670deacd06aedb90235e6bd873674c10467a05b4adcd62dbc019279efcbcbb11Ù!2026-09-13T15:14:36.4120244+00:00Ä‘´written before cut 2 \ No newline at end of file diff --git a/tests/vectors/document-variants-c0/legacy-catalog-trailing.msgpack b/tests/vectors/document-variants-c0/legacy-catalog-trailing.msgpack new file mode 100644 index 00000000..2755f37c --- /dev/null +++ b/tests/vectors/document-variants-c0/legacy-catalog-trailing.msgpack @@ -0,0 +1,2 @@ +“²cultcache.store.v1’—ÙGsha256:88d3fdf0a927acf3b163940d8f8c7fe62b3316542ce771a67ec8bc038f594788µfixtures.precut2.item¸fixtures.precut2.item.v1Úp{"schemaName":"fixtures.precut2.item","schemaVersion":"fixtures.precut2.item.v1","members":[{"slot":0,"name":"Name","type":"System.String","isReference":false,"many":false,"targetSchemaName":null,"indexAlias":null,"isName":true},{"slot":1,"name":"Count","type":"System.Int32","isReference":false,"many":false,"targetSchemaName":null,"indexAlias":null,"isName":false}]}ÀÙGsha256:dadf30bc920849ee89ed5d70cc73027070dd7571c5f878db1901f3abc459cc0c—ÙGsha256:670deacd06aedb90235e6bd873674c10467a05b4adcd62dbc019279efcbcbb11µfixtures.precut2.note¸fixtures.precut2.note.v1Ùç{"schemaName":"fixtures.precut2.note","schemaVersion":"fixtures.precut2.note.v1","members":[{"slot":0,"name":"Text","type":"System.String","isReference":false,"many":false,"targetSchemaName":null,"indexAlias":null,"isName":false}]}ÀÙGsha256:8552c4cd5a0a455fc43a37c2314d28617b7b9d36e43ee7ef2dd18cb148dbfe35“”ªitem:anvilÙGsha256:88d3fdf0a927acf3b163940d8f8c7fe62b3316542ce771a67ec8bc038f594788Ù!2026-09-13T15:14:36.4032226+00:00Ä’¥anvil”¬item:bellowsÙGsha256:88d3fdf0a927acf3b163940d8f8c7fe62b3316542ce771a67ec8bc038f594788Ù!2026-09-13T15:14:36.4114746+00:00Ä +’§bellows*”«note:originÙGsha256:670deacd06aedb90235e6bd873674c10467a05b4adcd62dbc019279efcbcbb11Ù!2026-09-13T15:14:36.4120244+00:00Ä‘´written before cut 2 \ No newline at end of file diff --git a/tests/vectors/document-variants-c2a/README.md b/tests/vectors/document-variants-c2a/README.md new file mode 100644 index 00000000..a9d5b98b --- /dev/null +++ b/tests/vectors/document-variants-c2a/README.md @@ -0,0 +1,12 @@ +# document-variants-c2a vectors + +`c2a-written-ids.msgpack` and `c2a-written-ids-directory/` are what `C2aGateWriter` writes: one record holding element ids, as a +single-file store (header v3) and as a directory store (header v5). The writer stamps every record with one fixed clock, so the +bytes are the same on every run, and `C2aGateWriter.TheCommittedVectorsAreWhatC2aWrites` compares the live output to these files +byte for byte (the directory as relative path and bytes; `*.lock` files hold no state and are left out). + +Regenerate them only when C2a's format moves on purpose: +`CULTLIB_GATE_OUT= dotnet test tests/GameCult.Caching.Tests --filter FullyQualifiedName~C2aStamps`, then copy the single +file and the directory store (the page under `store.cc.records/` is named by its content hash) over these. + +`old-reader-refusal.sh` is the gate that reads them with the pinned older readers; its header says how to run it. diff --git a/tests/vectors/document-variants-c2a/c2a-written-ids-directory/store.cc b/tests/vectors/document-variants-c2a/c2a-written-ids-directory/store.cc new file mode 100644 index 00000000..05040c81 Binary files /dev/null and b/tests/vectors/document-variants-c2a/c2a-written-ids-directory/store.cc differ diff --git a/tests/vectors/document-variants-c2a/c2a-written-ids-directory/store.cc.records/.commit.lock b/tests/vectors/document-variants-c2a/c2a-written-ids-directory/store.cc.records/.commit.lock new file mode 100644 index 00000000..e69de29b diff --git a/tests/vectors/document-variants-c2a/c2a-written-ids-directory/store.cc.records/578e09fc6a9cafb49d3eea249d7a5e1f861f36aa7dafe23f0605a87393403722.msgpack b/tests/vectors/document-variants-c2a/c2a-written-ids-directory/store.cc.records/578e09fc6a9cafb49d3eea249d7a5e1f861f36aa7dafe23f0605a87393403722.msgpack new file mode 100644 index 00000000..0012b9d8 Binary files /dev/null and b/tests/vectors/document-variants-c2a/c2a-written-ids-directory/store.cc.records/578e09fc6a9cafb49d3eea249d7a5e1f861f36aa7dafe23f0605a87393403722.msgpack differ diff --git a/tests/vectors/document-variants-c2a/c2a-written-ids.msgpack b/tests/vectors/document-variants-c2a/c2a-written-ids.msgpack new file mode 100644 index 00000000..038fe60c Binary files /dev/null and b/tests/vectors/document-variants-c2a/c2a-written-ids.msgpack differ diff --git a/tests/vectors/document-variants-c2a/old-reader-probes/OldReaderGateProbe.cs b/tests/vectors/document-variants-c2a/old-reader-probes/OldReaderGateProbe.cs new file mode 100644 index 00000000..c9390d85 --- /dev/null +++ b/tests/vectors/document-variants-c2a/old-reader-probes/OldReaderGateProbe.cs @@ -0,0 +1,164 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Threading.Tasks; +using GameCult.Caching; +using GameCult.Caching.MessagePack; +using MessagePack; +using NUnit.Framework; + +namespace GameCult.Caching.Tests +{ + // Copied into a checkout of cultlib-unity-v2.0.1 (b8fc5eb0) by old-reader-refusal.sh. Not part of this tree's suite. + // The 2.0.1 reader meets the stores C2a writes. The source is "committed" (tests/vectors/document-variants-c2a) or, when + // CULTLIB_GATE_OUT is set, "live" (what C2aGateWriter just wrote at the tree under test). Every refusal names the format, and + // the store is byte-identical after it. + public class OldReaderGateProbe + { + private const string SingleName = "c2a-written-ids.msgpack"; + private const string DirectoryName = "c2a-written-ids-directory"; + private const string ManifestName = "store.cc"; + + [CultDocument("tests.gate_probe", "tests.gate_probe.v1")] + [MessagePackObject(AllowPrivate = true)] + internal sealed class GateProbeDocument + { + [Key(0)] + [CultName] + public string Name = string.Empty; + } + + public static IEnumerable Sources() + { + yield return "committed"; + if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("CULTLIB_GATE_OUT"))) + yield return "live"; + } + + private static string SourceRoot(string source) => source == "live" + ? Environment.GetEnvironmentVariable("CULTLIB_GATE_OUT")! + : Path.Combine(Environment.GetEnvironmentVariable("CULTLIB_ROOT")!, "tests", "vectors", "document-variants-c2a"); + + private string _temp = string.Empty; + + [SetUp] + public void CreateTemp() + { + _temp = Path.Combine(Path.GetTempPath(), $"cultlib-gate-{Guid.NewGuid():N}"); + Directory.CreateDirectory(_temp); + } + + [TearDown] + public void DeleteTemp() + { + if (Directory.Exists(_temp)) + Directory.Delete(_temp, recursive: true); + } + + private string CopySingle(string source, string name) + { + var file = Path.Combine(_temp, name); + File.Copy(Path.Combine(SourceRoot(source), SingleName), file); + return file; + } + + private string CopyDirectory(string source, string name) + { + var root = Path.Combine(_temp, name); + CopyTree(Path.Combine(SourceRoot(source), DirectoryName), root); + return Path.Combine(root, ManifestName); + } + + private static void CopyTree(string from, string to) + { + Directory.CreateDirectory(to); + foreach (var file in Directory.GetFiles(from)) + File.Copy(file, Path.Combine(to, Path.GetFileName(file))); + foreach (var directory in Directory.GetDirectories(from)) + CopyTree(directory, Path.Combine(to, Path.GetFileName(directory))); + } + + // Every file under the root with its content hash, in a fixed order: a tree compares equal only if nothing in it moved. + private static string[] Tree(string root) => + Directory.GetFiles(root, "*", SearchOption.AllDirectories) + .OrderBy(file => file, StringComparer.Ordinal) + .Select(file => Path.GetRelativePath(root, file) + " " + Convert.ToHexString(SHA256.HashData(File.ReadAllBytes(file)))) + .ToArray(); + + private static void Names(Exception? error, string format, string what, string source) + { + Assert.That(error, Is.Not.Null, what); + TestContext.Out.WriteLine($"OLD-READER csharp refused {what} [{source}]: " + error!.Message); + Assert.That(error.ToString(), Does.Contain(format), what); + } + + [TestCaseSource(nameof(Sources))] + public void V3IsRefusedAtAttach(string source) + { + var file = CopySingle(source, "attach.cc"); + var before = File.ReadAllBytes(file); + using var cache = new CultCache(); + Names(Assert.Catch(() => cache.AddBackingStore(new SingleFileMessagePackBackingStore(file))), "cultcache.store.v3", "attach", source); + Assert.That(File.ReadAllBytes(file), Is.EqualTo(before)); + } + + [TestCaseSource(nameof(Sources))] + public void V3IsRefusedByPushAllOfAStoreThatWasNeverAttached(string source) + { + var file = CopySingle(source, "detached.cc"); + var before = File.ReadAllBytes(file); + var store = new SingleFileMessagePackBackingStore(file); + Names(Assert.Catch(() => store.PushAll()), "cultcache.store.v3", "PushAll of a detached store", source); + Assert.That(File.ReadAllBytes(file), Is.EqualTo(before)); + } + + [TestCaseSource(nameof(Sources))] + public async Task V3SwappedInAfterAttachIsRefusedByFlushAndPushAll(string source) + { + var file = Path.Combine(_temp, "swap.cc"); + using var cache = new CultCache(); + var store = new SingleFileMessagePackBackingStore(file); + cache.AddBackingStore(store); + await cache.AddAsync(new GateProbeDocument { Name = "staged" }); + File.Copy(Path.Combine(SourceRoot(source), SingleName), file, overwrite: true); + var before = File.ReadAllBytes(file); + + Names(Assert.Catch(() => cache.FlushAllBackingStores()), "cultcache.store.v3", "flush after a swap", source); + Assert.That(File.ReadAllBytes(file), Is.EqualTo(before)); + Names(Assert.Catch(() => store.PushAll()), "cultcache.store.v3", "PushAll after a swap", source); + Assert.That(File.ReadAllBytes(file), Is.EqualTo(before)); + } + + [TestCaseSource(nameof(Sources))] + public void V5IsRefusedByTheDirectoryStoreAtAttach(string source) + { + var manifest = CopyDirectory(source, "attach-directory"); + var before = Tree(Path.GetDirectoryName(manifest)!); + using var cache = new CultCache(); + Names(Assert.Catch(() => cache.AddBackingStore(new DirectoryMessagePackBackingStore(manifest))), "cultcache.store.v5", "directory attach", source); + Assert.That(Tree(Path.GetDirectoryName(manifest)!), Is.EqualTo(before)); + } + + [TestCaseSource(nameof(Sources))] + public async Task V5SwappedInAfterAttachIsRefusedByTheDirectoryStoreAtFlush(string source) + { + var root = Path.Combine(_temp, "swap-directory"); + Directory.CreateDirectory(root); + var manifest = Path.Combine(root, ManifestName); + using var cache = new CultCache(); + var store = new DirectoryMessagePackBackingStore(manifest); + cache.AddBackingStore(store); + await cache.AddAsync(new GateProbeDocument { Name = "staged" }); + CopyTree(Path.Combine(SourceRoot(source), DirectoryName), root); + var before = Tree(root); + + Names(Assert.Catch(() => cache.FlushAllBackingStores()), "cultcache.store.v5", "directory flush after a swap", source); + Assert.That(Tree(root), Is.EqualTo(before)); + Names(Assert.Catch(() => store.PushAll()), "cultcache.store.v5", "directory PushAll after a swap", source); + Assert.That(Tree(root), Is.EqualTo(before)); + } + } +} diff --git a/tests/vectors/document-variants-c2a/old-reader-probes/OldReaderRefListProbe.cs b/tests/vectors/document-variants-c2a/old-reader-probes/OldReaderRefListProbe.cs new file mode 100644 index 00000000..6c6cb029 --- /dev/null +++ b/tests/vectors/document-variants-c2a/old-reader-probes/OldReaderRefListProbe.cs @@ -0,0 +1,64 @@ +#nullable enable +using System; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using GameCult.Caching; +using GameCult.Caching.MessagePack; +using NUnit.Framework; +using RefListHolder = GameCult.Caching.Tests.RefListDriftTests.RefListHolder; +using RefListTarget = GameCult.Caching.Tests.RefListDriftTests.RefListTarget; + +namespace GameCult.Caching.Tests +{ + // Copied into a checkout of cultlib-unity-v2.0.1 (b8fc5eb0) by old-reader-refusal.sh. Not part of this tree's suite. + // It attaches the store the 1.0.60 writer produced and prints the REFLIST-SUMMARY line that + // RefListDriftTests.A_1_0_60_Written_Store_Opens_Like_2_0_1 prints at head; 'gate csharp' diffs the two lines. The document + // types are 2.0.1's own RefListDriftTests.RefListTarget and RefListHolder. + public class OldReaderRefListProbe + { + // RefListDriftTests.A_1_0_60_Written_Store_Opens_Like_2_0_1 is this test's body at head; the two must stay the same code. + [Test] + public async Task A_1_0_60_Written_Store_Attaches_As_Compatible_Drift() + { + var copy = Path.Combine(Path.GetTempPath(), $"reflist-gate-{Guid.NewGuid():N}.cc"); + File.Copy(Path.Combine(Environment.GetEnvironmentVariable("CULTLIB_GATE_OUT")!, "v1-0-60-ref-list.cc"), copy); + try + { + string[] warnings; + using (var cache = new CultCache()) + { + var store = new SingleFileMessagePackBackingStore(copy); + cache.AddBackingStore(store); + await cache.PullAllBackingStoresAsync(); + warnings = store.LastSchemaMigrationReports.SelectMany(report => report.Warnings).Select(warning => warning.Code).Distinct().OrderBy(code => code, StringComparer.Ordinal).ToArray(); + Assert.That(warnings, Does.Contain("gained_reference_semantics")); + var holder = cache.GetAll().Single(); + Assert.That(holder.Items.Select(item => item.Key.Value), Is.EqualTo(new[] { "alpha", "beta" })); + + // The write both runtimes make: the targets as they are, the holder with its list reversed. + foreach (var target in cache.GetAll().ToArray()) await cache.UpsertAsync(target, cache.TryGetHandle(target)); + holder.Items = holder.Items.AsEnumerable().Reverse().ToArray(); + await cache.UpsertAsync(holder, cache.TryGetHandle(holder)); + await cache.FlushAsync(); + } + + var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(copy)); + var records = snapshot.Records.OrderBy(record => record.Key, StringComparer.Ordinal).Select(record => record.Key + "=" + record.SchemaId).ToArray(); + var listId = snapshot.Records.Single(record => record.Key == "list").SchemaId; + var slot = snapshot.SchemaCatalog.Single(entry => entry.SchemaId == listId).Members.Single(member => member.MemberName == "Items"); + using var reopened = new CultCache(); + reopened.AddBackingStore(new SingleFileMessagePackBackingStore(copy)); + await reopened.PullAllBackingStoresAsync(); + var items = reopened.GetAll().Single().Items.Select(item => item.Key.Value).ToArray(); + Assert.That(items, Is.EqualTo(new[] { "beta", "alpha" })); + Assert.That(reopened.Get(reopened.GetAll().Single().Items[0].Key)!.Name, Is.EqualTo("beta")); + TestContext.Out.WriteLine("REFLIST-SUMMARY " + System.Text.Json.JsonSerializer.Serialize(new { records, items, itemsSlot = new { slot.IsReference, slot.TargetSchemaName }, warnings, header = snapshot.FormatVersion })); + } + finally + { + File.Delete(copy); + } + } + } +} diff --git a/tests/vectors/document-variants-c2a/old-reader-probes/old-reader-v3.test.ts b/tests/vectors/document-variants-c2a/old-reader-probes/old-reader-v3.test.ts new file mode 100644 index 00000000..083ad75a --- /dev/null +++ b/tests/vectors/document-variants-c2a/old-reader-probes/old-reader-v3.test.ts @@ -0,0 +1,45 @@ +import assert from "node:assert/strict"; +import { mkdtemp, readFile, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { test } from "node:test"; + +import { documentPayloadFromBytes } from "../src/document"; +import { SingleFileMessagePackBackingStore } from "../src/single-file-messagepack-backing-store"; + +// Copied into a checkout of cultcache-rs-v0.3.1 (02cf04b5) by old-reader-refusal.sh. Not part of this tree's suite. +// The vector is the store C2aGateWriter wrote through CultCacheMessagePack.Create. +const vector = resolve(__dirname, "../../../../tests/vectors/document-variants-c2a/c2a-written-ids.msgpack"); +const envelope = { key: "probe", type: "probe", payload: documentPayloadFromBytes(new Uint8Array([0x80])), storedAt: "2026-10-22T00:00:00Z" }; + +async function copyOfTheVector(): Promise<{ file: string; bytes: Buffer }> { + const bytes = await readFile(vector); + const file = join(await mkdtemp(join(tmpdir(), "cultcache-oldreader-")), "store.msgpack"); + await writeFile(file, bytes); + return { file, bytes }; +} + +function refusesV3(label: string) { + return (error: Error) => { + console.log(`OLD-READER typescript refused ${label}: ` + error.message); + return /cultcache\.store\.v3/u.test(error.message); + }; +} + +test("old TypeScript reader refuses the C2a element-id store at read", async () => { + const { file, bytes } = await copyOfTheVector(); + await assert.rejects(() => new SingleFileMessagePackBackingStore(file).pullAll(), refusesV3("pullAll")); + assert.equal(Buffer.compare(await readFile(file), bytes), 0); +}); + +test("old TypeScript reader refuses a per-record push to the C2a element-id store", async () => { + const { file, bytes } = await copyOfTheVector(); + await assert.rejects(() => new SingleFileMessagePackBackingStore(file).push(envelope), refusesV3("push")); + assert.equal(Buffer.compare(await readFile(file), bytes), 0); +}); + +test("old TypeScript reader refuses a whole-store pushAll over the C2a element-id store", async () => { + const { file, bytes } = await copyOfTheVector(); + await assert.rejects(() => new SingleFileMessagePackBackingStore(file).pushAll([envelope]), refusesV3("pushAll")); + assert.equal(Buffer.compare(await readFile(file), bytes), 0); +}); diff --git a/tests/vectors/document-variants-c2a/old-reader-probes/old_reader_v3.rs b/tests/vectors/document-variants-c2a/old-reader-probes/old_reader_v3.rs new file mode 100644 index 00000000..0965eff2 --- /dev/null +++ b/tests/vectors/document-variants-c2a/old-reader-probes/old_reader_v3.rs @@ -0,0 +1,53 @@ +// Copied into a checkout of cultcache-rs-v0.3.1 (02cf04b5) by old-reader-refusal.sh. Not part of this tree's suite. +// The vector is the store C2aGateWriter wrote through CultCacheMessagePack.Create. +use cultcache_rs::{CacheBackingStore, CultCacheEnvelope, PushAllOptions, SingleFileMessagePackBackingStore}; + +fn copy_of_the_vector() -> (tempfile::TempDir, std::path::PathBuf, Vec) { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("store.msgpack"); + let vector = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../tests/vectors/document-variants-c2a/c2a-written-ids.msgpack"); + std::fs::copy(vector, &path).unwrap(); + let bytes = std::fs::read(&path).unwrap(); + (temp, path, bytes) +} + +fn envelope() -> CultCacheEnvelope { + CultCacheEnvelope { + key: "probe".into(), + r#type: "probe".into(), + payload: vec![0x80], + stored_at: "2026-10-22T00:00:00Z".into(), + schema_id: None, + } +} + +fn refused_v3(label: &str, error: anyhow::Error) { + let message = format!("{error:#}"); + println!("OLD-READER rust refused {label}: {message}"); + assert!(message.contains("cultcache.store.v3"), "{message}"); +} + +#[test] +fn old_rust_reader_refuses_the_c2a_element_id_store_at_read() { + let (_temp, path, bytes) = copy_of_the_vector(); + refused_v3("pull_all", SingleFileMessagePackBackingStore::new(&path).pull_all().unwrap_err()); + assert_eq!(std::fs::read(&path).unwrap(), bytes); +} + +#[test] +fn old_rust_reader_refuses_a_per_record_push_to_the_c2a_element_id_store() { + let (_temp, path, bytes) = copy_of_the_vector(); + refused_v3("push", SingleFileMessagePackBackingStore::new(&path).push(&envelope()).unwrap_err()); + assert_eq!(std::fs::read(&path).unwrap(), bytes); +} + +#[test] +fn old_rust_reader_refuses_a_whole_store_push_all_over_the_c2a_element_id_store() { + let (_temp, path, bytes) = copy_of_the_vector(); + let error = SingleFileMessagePackBackingStore::new(&path) + .push_all(std::slice::from_ref(&envelope()), PushAllOptions::default()) + .unwrap_err(); + refused_v3("push_all", error); + assert_eq!(std::fs::read(&path).unwrap(), bytes); +} diff --git a/tests/vectors/document-variants-c2a/old-reader-probes/test_old_reader_v3.py b/tests/vectors/document-variants-c2a/old-reader-probes/test_old_reader_v3.py new file mode 100644 index 00000000..83a7e2ea --- /dev/null +++ b/tests/vectors/document-variants-c2a/old-reader-probes/test_old_reader_v3.py @@ -0,0 +1,41 @@ +import tempfile +import unittest +from pathlib import Path + +from cultcache_py import CultCacheEnvelope, SingleFileMessagePackBackingStore + + +# Copied into a checkout of cultcache-rs-v0.3.1 (02cf04b5) by old-reader-refusal.sh. Not part of this tree's suite. +# The vector is the store C2aGateWriter wrote through CultCacheMessagePack.Create. +class OldReaderV3(unittest.TestCase): + def setUp(self) -> None: + vector = Path(__file__).resolve().parents[3] / "tests" / "vectors" / "document-variants-c2a" / "c2a-written-ids.msgpack" + self._tmp = tempfile.TemporaryDirectory() + self.addCleanup(self._tmp.cleanup) + self.path = Path(self._tmp.name) / "store.msgpack" + self.bytes = vector.read_bytes() + self.path.write_bytes(self.bytes) + self.envelope = CultCacheEnvelope(key="probe", type="probe", payload=b"\x80", stored_at="2026-10-22T00:00:00Z") + + def refuses_v3(self, label: str, action) -> None: + with self.assertRaises(ValueError) as caught: + action(SingleFileMessagePackBackingStore(self.path)) + print(f"OLD-READER python refused {label}:", caught.exception) + self.assertIn("cultcache.store.v3", str(caught.exception)) + self.assertEqual(self.path.read_bytes(), self.bytes) + + def test_old_python_reader_refuses_the_c2a_element_id_store_at_read(self) -> None: + self.refuses_v3("pull_all", lambda store: store.pull_all()) + + def test_old_python_reader_refuses_a_per_record_push(self) -> None: + self.refuses_v3("push", lambda store: store.push(self.envelope)) + + def test_old_python_reader_refuses_a_whole_store_push_all(self) -> None: + self.refuses_v3("push_all", lambda store: store.push_all([self.envelope])) + + def test_old_python_reader_refuses_a_delete(self) -> None: + self.refuses_v3("delete", lambda store: store.delete("probe", "probe")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/vectors/document-variants-c2a/old-reader-refusal.sh b/tests/vectors/document-variants-c2a/old-reader-refusal.sh new file mode 100755 index 00000000..bfafa01d --- /dev/null +++ b/tests/vectors/document-variants-c2a/old-reader-refusal.sh @@ -0,0 +1,220 @@ +#!/usr/bin/env bash +# The C2a format gate: a reader from before element ids must refuse what C2a writes, and a store the 1.0.60 writer made must +# open alike at 2.0.1 and at the tree under test. Each old revision is a resolved SHA with its tag beside it, because the +# verification clone holds ancestors, not tags; an environment variable overrides each. +# +# C# OLD_READER_REV_CSHARP b8fc5eb0 cultlib-unity-v2.0.1 +# TypeScript, OLD_READER_REV_LEAF 02cf04b5 cultcache-rs-v0.3.1 (the whole-store writer fix; no cultcache-ts tag carries it) +# Python, Rust +# 1.0.60 writer OLD_WRITER_REV 45c2f400 cultlib-unity-v1.0.60 +# +# old-reader-refusal.sh stage csharp|typescript|python|rust|writer +# Run in a CultLib clone. Commits the old revision plus the C2a-written vectors and that runtime's probes onto a scratch +# ref and prints its SHA. The tree at that SHA is the old code with extra tests ("writer" is the 1.0.60 writer). +# old-reader-refusal.sh probe csharp|typescript|python|rust|writer +# Run at the root of a staged tree, with the runtime's toolchain on PATH. Exit status zero means every probe passed; the +# refusal messages are printed. +# old-reader-refusal.sh gate csharp|typescript|python|rust +# Run in a CultLib clone at the revision under test: stage, then probe. For csharp, C2aGateWriter first writes the live +# v3 and v5 stores into $GATE_OUT (default /tmp/gate) and the 2.0.1 probes read them beside the committed vectors; the +# 1.0.60 writer then writes a ref-list store there, 2.0.1 and the tree under test both open it, write the same documents +# back and reread the file, and their REFLIST-SUMMARY lines (records by schema id, the list's items in order, the Items +# slot's catalog entry, warnings, header) must be identical. +# +# Every stage asserts how many cases it PASSED, so a probe that was renamed out of its filter or a leaf test that was skipped +# fails the gate by name ('-count') instead of passing with nothing run; a C# result that is not Passed (an [Ignore]d +# case is NotExecuted) fails '-notpassed' and Python's skipped/expected-failure forms fail 'python-notpassed'. The expected counts are the constants below; +# each is confirmed against the probe's source: C#, the Passed results in the TRX the run wrote (by class); Rust, the sum of 'N passed'; +# Python, 'Ran N tests'; TypeScript, node's 'pass N'. Each stage also counts the refusal lines it printed (EXPECT_REFUSED_*). +set -euo pipefail + +here="tests/vectors/document-variants-c2a" +self="$(cd "$(dirname "$0")" && pwd)/$(basename "$0")" +OLD_READER_REV_CSHARP="${OLD_READER_REV_CSHARP:-b8fc5eb0}" # cultlib-unity-v2.0.1 +OLD_READER_REV_LEAF="${OLD_READER_REV_LEAF:-02cf04b5}" # cultcache-rs-v0.3.1 +OLD_WRITER_REV="${OLD_WRITER_REV:-45c2f400}" # cultlib-unity-v1.0.60 + +EXPECT_C2A_WRITER=2 # C2aGateWriter: C2aStamps..., TheCommittedVectorsAreWhatC2aWrites +EXPECT_READER_GATE=10 # OldReaderGateProbe: five TestCaseSource tests, each over the committed and the live store +EXPECT_READER_REFLIST=1 # OldReaderRefListProbe +EXPECT_OLD_WRITER=1 # OldWriterRefListProbe +EXPECT_HEAD_REFLIST=1 # RefListDriftTests.A_1_0_60_Written_Store_Opens_Like_2_0_1 +EXPECT_RUST=3 # tests/old_reader_v3.rs +EXPECT_PYTHON=4 # test_old_reader_v3.py +EXPECT_TYPESCRIPT=3 # old-reader-v3.test.ts + +# Refusals each stage must PRINT: every probe writes one 'OLD-READER refused