diff --git a/persistence/binary/src/main/java/org/eclipse/serializer/persistence/binary/org/eclipse/serializer/reference/BinaryHandlerLazyDefault.java b/persistence/binary/src/main/java/org/eclipse/serializer/persistence/binary/org/eclipse/serializer/reference/BinaryHandlerLazyDefault.java
index c4d38076..2f4d301d 100644
--- a/persistence/binary/src/main/java/org/eclipse/serializer/persistence/binary/org/eclipse/serializer/reference/BinaryHandlerLazyDefault.java
+++ b/persistence/binary/src/main/java/org/eclipse/serializer/persistence/binary/org/eclipse/serializer/reference/BinaryHandlerLazyDefault.java
@@ -107,6 +107,12 @@ public final void store(
"Persisting it would result in data loss."
);
}
+
+ if(referenceOid != 0)
+ {
+ // cached id of an unloaded referent: referenced but not stored in this commit.
+ handler.noteTrustedReference(referenceOid);
+ }
}
else
{
diff --git a/persistence/binary/src/main/java/org/eclipse/serializer/persistence/binary/types/Binary.java b/persistence/binary/src/main/java/org/eclipse/serializer/persistence/binary/types/Binary.java
index 80d3992d..fa57de14 100644
--- a/persistence/binary/src/main/java/org/eclipse/serializer/persistence/binary/types/Binary.java
+++ b/persistence/binary/src/main/java/org/eclipse/serializer/persistence/binary/types/Binary.java
@@ -469,6 +469,14 @@ static final void setEntityHeaderRawValuesToAddress(
long address;
private HelperEntry helperEntry;
+
+ /**
+ * Transient store-transport metadata, never part of the persisted data itself: the object ids this
+ * data references without containing the corresponding entities, trusting that they already exist
+ * in the persistence target. Set by the storer on commit; the target may validate their existence
+ * before committing the data.
+ */
+ private long[] trustedObjectIds;
@@ -2791,10 +2799,44 @@ void storeRangeToAddress(final long address, final long sourceAddress, final lon
}
+ /**
+ * Sets the transient store-transport metadata of object ids referenced but not contained by this data.
+ *
+ * COMMIT-scope metadata, not per-chunk state: it is set exactly once, on the representative
+ * {@code Binary} instance the storer passes to {@code PersistenceTarget#write}, and covers the
+ * ENTIRE commit across all channels. Consumers read it from that instance and partition per
+ * channel themselves; per-channel chunk views deliberately do not carry it.
+ *
+ * @param trustedObjectIds the referenced object ids whose entities are trusted to already exist in the target.
+ *
+ * @see #trustedObjectIds()
+ */
+ public final void setTrustedObjectIds(final long[] trustedObjectIds)
+ {
+ this.trustedObjectIds = trustedObjectIds;
+ }
+
+ /**
+ * The object ids this data references without containing the corresponding entities, i.e. ids whose
+ * entities the storer trusted to already exist in the persistence target. {@code null} if no such ids
+ * were collected (capturing disabled or nothing to report).
+ *
+ * Commit-scope metadata: only present on the representative instance passed to
+ * {@code PersistenceTarget#write}, covering the entire commit across all channels
+ * (see {@link #setTrustedObjectIds(long[])}).
+ *
+ * @return the trusted object ids, or {@code null}.
+ */
+ public final long[] trustedObjectIds()
+ {
+ return this.trustedObjectIds;
+ }
+
+
///////////////////////////////////////////////////////////////////////////
// Helper //
///////////
-
+
/**
* Helper instances can be used as temporary additional state for the duration of the building process.
* E.g.: JDK hash collections cannot properly collect elements during the building process as the element instances
diff --git a/persistence/binary/src/main/java/org/eclipse/serializer/persistence/binary/types/BinaryStorer.java b/persistence/binary/src/main/java/org/eclipse/serializer/persistence/binary/types/BinaryStorer.java
index e1ae4b4f..f95480e3 100644
--- a/persistence/binary/src/main/java/org/eclipse/serializer/persistence/binary/types/BinaryStorer.java
+++ b/persistence/binary/src/main/java/org/eclipse/serializer/persistence/binary/types/BinaryStorer.java
@@ -15,6 +15,7 @@
*/
import org.eclipse.serializer.collections.BulkList;
+import org.eclipse.serializer.collections.Set_long;
import org.eclipse.serializer.hashing.XHashing;
import org.eclipse.serializer.math.XMath;
import org.eclipse.serializer.persistence.exceptions.PersistenceException;
@@ -26,6 +27,7 @@
import org.slf4j.Logger;
import java.time.Duration;
+import java.util.Arrays;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
@@ -143,6 +145,15 @@ protected static int defaultSlotSize()
private int hashRange;
private long itemCount;
private long pinCount ; // subset of itemCount: pure retention entries (see PinItem), no store payload.
+
+ /*
+ * Object ids this storer writes into the data as references while trusting that the referenced
+ * entity already exists in the target (global registry hits and unloaded Lazy references' cached
+ * ids). Null when capturing is disabled, so the feature has zero overhead in that case.
+ * At commit, ids that are also stored by the commit itself are pruned; the remainder is attached
+ * to the written Binary for the target to validate. Guarded by objectRegistryMonitor.
+ */
+ private final Set_long trustedObjectIds;
private final BulkList commitListeners = BulkList.New(0);
@@ -179,6 +190,31 @@ protected Default(
final boolean switchByteOrder ,
final Persister persister
)
+ {
+ this(
+ objectManager ,
+ objectRetriever ,
+ typeManager ,
+ target ,
+ bufferSizeProvider,
+ channelCount ,
+ switchByteOrder ,
+ persister ,
+ false
+ );
+ }
+
+ protected Default(
+ final PersistenceObjectManager objectManager ,
+ final ObjectSwizzling objectRetriever ,
+ final PersistenceTypeHandlerManager typeManager ,
+ final PersistenceTarget target ,
+ final BufferSizeProviderIncremental bufferSizeProvider ,
+ final int channelCount ,
+ final boolean switchByteOrder ,
+ final Persister persister ,
+ final boolean captureTrustedObjectIds
+ )
{
super();
this.objectManager = notNull(objectManager) ;
@@ -190,6 +226,7 @@ protected Default(
this.chunksHashRange = channelCount - 1 ;
this.switchByteOrder = switchByteOrder ;
this.persister = mayNull(persister) ;
+ this.trustedObjectIds = captureTrustedObjectIds ? Set_long.New() : null;
this.defaultInitialize();
}
@@ -330,6 +367,11 @@ protected void internalInitialize(final int hashLength)
// must be clear instead of just reset to avoid memory leaks
this.commitListeners.clear();
this.persistenceObjectRegistrationListener.clear();
+
+ if(this.trustedObjectIds != null)
+ {
+ this.trustedObjectIds.clear();
+ }
}
}
@@ -681,6 +723,12 @@ public Object commit()
this.typeManager.checkForPendingRootInstances();
this.typeManager.checkForPendingRootsStoring(this);
writeData = this.synchComplete();
+
+ final long[] trustedObjectIds = this.synchYieldTrustedObjectIds();
+ if(trustedObjectIds != null)
+ {
+ writeData.setTrustedObjectIds(trustedObjectIds);
+ }
}
// very costly IO-operation does not need to occupy the lock
@@ -718,6 +766,55 @@ public Object commit()
return null;
}
+ /**
+ * Yields the trusted object ids of the current commit as an array, pruned by the ids the commit
+ * stores itself: an id that is both referenced and stored in the same commit is guaranteed, not
+ * trusted (e.g. {@code storeAll(parent, child)} re-storing a registry-known child, or an eager
+ * storer's items). Returns {@code null} if capturing is disabled or nothing remains after pruning.
+ *
+ * Must be called under {@code objectRegistryMonitor}.
+ */
+ private long[] synchYieldTrustedObjectIds()
+ {
+ if(this.trustedObjectIds == null || this.trustedObjectIds.isEmpty())
+ {
+ return null;
+ }
+
+ /*
+ * Presized to the exact item chain length to avoid rehashing: pins occupy hash slot
+ * entries (itemCount) but are never chained, so the chain holds itemCount - pinCount
+ * entries (skip items are chained but rare; their exclusion below costs no rebuild).
+ */
+ final Set_long storedObjectIds = Set_long.New(
+ XHashing.padHashLength(Math.max((int)(this.itemCount - this.pinCount), 1))
+ );
+ for(Item e = this.head; (e = e.next) != null;)
+ {
+ if(!isSkipItem(e))
+ {
+ storedObjectIds.add(e.oid);
+ }
+ }
+
+ final long[] buffer = new long[(int)this.trustedObjectIds.size()];
+ final int[] count = {0};
+ this.trustedObjectIds.iterate(objectId ->
+ {
+ if(!storedObjectIds.contains(objectId))
+ {
+ buffer[count[0]++] = objectId;
+ }
+ });
+
+ return count[0] == 0
+ ? null
+ : count[0] == buffer.length
+ ? buffer
+ : Arrays.copyOf(buffer, count[0])
+ ;
+ }
+
public final long lookupOid(final Object object)
{
/*
@@ -887,6 +984,10 @@ public void registerSkippedOptional(
*/
synchronized(this.objectRegistryMonitor)
{
+ // the skipped id is also a "trusted reference": referenced but not stored in this
+ // commit - record it so the persistence target can validate its existence.
+ this.noteTrustedReference(objectId);
+
// any existing local entry (regular, skip or pin) already holds the instance strongly
if(Swizzling.isFoundId(this.lookupOidLazyApplicable(instance)))
{
@@ -897,6 +998,22 @@ public void registerSkippedOptional(
}
}
+ @Override
+ public final void noteTrustedReference(final long objectId)
+ {
+ // only data object ids can dangle; TIDs/CIDs/null are resolved at runtime, not via entities.
+ if(this.trustedObjectIds == null || !Persistence.IdType.OID.isInRange(objectId))
+ {
+ return;
+ }
+
+ // reentrant from the registerSkippedOptional path; the Lazy handler path acquires it fresh.
+ synchronized(this.objectRegistryMonitor)
+ {
+ this.trustedObjectIds.add(objectId);
+ }
+ }
+
protected final long register(final Object instance)
{
/* Note:
@@ -1133,7 +1250,7 @@ public class Eager extends Default
final Persister persister
)
{
- super(
+ this(
objectManager ,
objectRetriever ,
typeManager ,
@@ -1141,7 +1258,33 @@ public class Eager extends Default
bufferSizeProvider,
channelCount ,
switchByteOrder ,
- persister
+ persister ,
+ false
+ );
+ }
+
+ Eager(
+ final PersistenceObjectManager objectManager ,
+ final ObjectSwizzling objectRetriever ,
+ final PersistenceTypeHandlerManager typeManager ,
+ final PersistenceTarget target ,
+ final BufferSizeProviderIncremental bufferSizeProvider ,
+ final int channelCount ,
+ final boolean switchByteOrder ,
+ final Persister persister ,
+ final boolean captureTrustedObjectIds
+ )
+ {
+ super(
+ objectManager ,
+ objectRetriever ,
+ typeManager ,
+ target ,
+ bufferSizeProvider ,
+ channelCount ,
+ switchByteOrder ,
+ persister ,
+ captureTrustedObjectIds
);
}
@@ -1233,27 +1376,29 @@ public final class Batching extends Default implements BatchStorer
private final Object commitLock = new Object();
Batching(
- final PersistenceObjectManager objectManager ,
- final ObjectSwizzling objectRetriever ,
- final PersistenceTypeHandlerManager typeManager ,
- final PersistenceTarget target ,
- final BufferSizeProviderIncremental bufferSizeProvider,
- final int channelCount ,
- final boolean switchByteOrder ,
- final Persister persister ,
- final BatchStorer.Controller controller ,
- final Duration checkInterval
+ final PersistenceObjectManager objectManager ,
+ final ObjectSwizzling objectRetriever ,
+ final PersistenceTypeHandlerManager typeManager ,
+ final PersistenceTarget target ,
+ final BufferSizeProviderIncremental bufferSizeProvider ,
+ final int channelCount ,
+ final boolean switchByteOrder ,
+ final Persister persister ,
+ final BatchStorer.Controller controller ,
+ final Duration checkInterval ,
+ final boolean captureTrustedObjectIds
)
{
super(
- objectManager ,
- objectRetriever ,
- typeManager ,
- target ,
- bufferSizeProvider,
- channelCount ,
- switchByteOrder ,
- persister
+ objectManager ,
+ objectRetriever ,
+ typeManager ,
+ target ,
+ bufferSizeProvider ,
+ channelCount ,
+ switchByteOrder ,
+ persister ,
+ captureTrustedObjectIds
);
this.controller = notNull(controller);
@@ -1645,10 +1790,32 @@ public static BinaryStorer.Creator Creator(
final BinaryChannelCountProvider channelCountProvider,
final boolean switchByteOrder
)
+ {
+ return Creator(channelCountProvider, switchByteOrder, false);
+ }
+
+ /**
+ * Creates a new default {@link BinaryStorer.Creator}.
+ *
+ * @param channelCountProvider supplies the number of channels each created storer will partition its
+ * chunk buffers across.
+ * @param switchByteOrder whether persisted values should use a non-native byte order.
+ * @param captureTrustedObjectIds whether created storers collect the object ids they reference without
+ * storing (see {@link Binary#trustedObjectIds()}) so the persistence
+ * target can validate their existence.
+ *
+ * @return the newly created storer creator.
+ */
+ public static BinaryStorer.Creator Creator(
+ final BinaryChannelCountProvider channelCountProvider ,
+ final boolean switchByteOrder ,
+ final boolean captureTrustedObjectIds
+ )
{
return new BinaryStorer.Creator.Default(
notNull(channelCountProvider),
- switchByteOrder
+ switchByteOrder ,
+ captureTrustedObjectIds
);
}
@@ -1703,8 +1870,9 @@ public abstract class Abstract implements BinaryStorer.Creator
////////////////////
- private final BinaryChannelCountProvider channelCountProvider;
- private final boolean switchByteOrder ;
+ private final BinaryChannelCountProvider channelCountProvider ;
+ private final boolean switchByteOrder ;
+ private final boolean captureTrustedObjectIds;
@@ -1716,28 +1884,43 @@ protected Abstract(
final BinaryChannelCountProvider channelCountProvider,
final boolean switchByteOrder
)
+ {
+ this(channelCountProvider, switchByteOrder, false);
+ }
+
+ protected Abstract(
+ final BinaryChannelCountProvider channelCountProvider ,
+ final boolean switchByteOrder ,
+ final boolean captureTrustedObjectIds
+ )
{
super();
- this.channelCountProvider = channelCountProvider;
- this.switchByteOrder = switchByteOrder ;
+ this.channelCountProvider = channelCountProvider ;
+ this.switchByteOrder = switchByteOrder ;
+ this.captureTrustedObjectIds = captureTrustedObjectIds;
}
-
-
+
+
///////////////////////////////////////////////////////////////////////////
// methods //
////////////
-
+
protected int channelCount()
{
return this.channelCountProvider.getChannelCount();
}
-
+
protected boolean switchByteOrder()
{
return this.switchByteOrder;
}
+ protected boolean captureTrustedObjectIds()
+ {
+ return this.captureTrustedObjectIds;
+ }
+
}
/**
@@ -1753,7 +1936,16 @@ public final class Default extends Abstract
final boolean switchByteOrder
)
{
- super(channelCountProvider, switchByteOrder);
+ this(channelCountProvider, switchByteOrder, false);
+ }
+
+ Default(
+ final BinaryChannelCountProvider channelCountProvider ,
+ final boolean switchByteOrder ,
+ final boolean captureTrustedObjectIds
+ )
+ {
+ super(channelCountProvider, switchByteOrder, captureTrustedObjectIds);
}
@Override
@@ -1769,14 +1961,15 @@ public final BinaryStorer createLazyStorer(
this.validateIsStoring(target);
final BinaryStorer.Default storer = new BinaryStorer.Default(
- objectManager ,
- objectRetriever ,
- typeManager ,
- target ,
- bufferSizeProvider ,
- this.channelCount() ,
- this.switchByteOrder(),
- persister
+ objectManager ,
+ objectRetriever ,
+ typeManager ,
+ target ,
+ bufferSizeProvider ,
+ this.channelCount() ,
+ this.switchByteOrder() ,
+ persister ,
+ this.captureTrustedObjectIds()
);
objectManager.registerLocalRegistry(storer);
@@ -1795,14 +1988,15 @@ public BinaryStorer createEagerStorer(
this.validateIsStoring(target);
final BinaryStorer.Eager storer = new BinaryStorer.Eager(
- objectManager ,
- objectRetriever ,
- typeManager ,
- target ,
- bufferSizeProvider ,
- this.channelCount() ,
- this.switchByteOrder(),
- persister
+ objectManager ,
+ objectRetriever ,
+ typeManager ,
+ target ,
+ bufferSizeProvider ,
+ this.channelCount() ,
+ this.switchByteOrder() ,
+ persister ,
+ this.captureTrustedObjectIds()
);
objectManager.registerLocalRegistry(storer);
@@ -1824,16 +2018,17 @@ public BatchStorer createBatchStorer(
this.validateIsStoring(target);
final BinaryStorer.Batching storer = new BinaryStorer.Batching(
- objectManager ,
- objectRetriever ,
- typeManager ,
- target ,
- bufferSizeProvider ,
- this.channelCount() ,
- this.switchByteOrder(),
- persister ,
- controller ,
- checkInterval
+ objectManager ,
+ objectRetriever ,
+ typeManager ,
+ target ,
+ bufferSizeProvider ,
+ this.channelCount() ,
+ this.switchByteOrder() ,
+ persister ,
+ controller ,
+ checkInterval ,
+ this.captureTrustedObjectIds()
);
objectManager.registerLocalRegistry(storer);
diff --git a/persistence/persistence/src/main/java/org/eclipse/serializer/persistence/types/PersistenceObjectIdRequestor.java b/persistence/persistence/src/main/java/org/eclipse/serializer/persistence/types/PersistenceObjectIdRequestor.java
index f68b522d..408de3bd 100644
--- a/persistence/persistence/src/main/java/org/eclipse/serializer/persistence/types/PersistenceObjectIdRequestor.java
+++ b/persistence/persistence/src/main/java/org/eclipse/serializer/persistence/types/PersistenceObjectIdRequestor.java
@@ -94,6 +94,9 @@ public void registerEagerOptional(
* instance's object registry entry gets reaped and the storage garbage collector can delete the
* referenced entity while the chunk referencing it is not yet committed, so the commit would persist
* a dangling reference (missing entity on later loads).
+ *
+ * Storers may additionally record the object id to have its existence validated by the persistence
+ * target before the data referencing it is committed (see the trusted-id validation).
*
* @param the instance type.
* @param objectId the object id the instance is already registered with.
diff --git a/persistence/persistence/src/main/java/org/eclipse/serializer/persistence/types/PersistenceObjectManager.java b/persistence/persistence/src/main/java/org/eclipse/serializer/persistence/types/PersistenceObjectManager.java
index ae1c6ef1..65597f2f 100644
--- a/persistence/persistence/src/main/java/org/eclipse/serializer/persistence/types/PersistenceObjectManager.java
+++ b/persistence/persistence/src/main/java/org/eclipse/serializer/persistence/types/PersistenceObjectManager.java
@@ -324,6 +324,8 @@ public long ensureObjectId(
* can drop its last strong reference between store and commit, the registry's weak entry
* gets reaped and the storage GC deletes the entity while the chunk referencing it is
* not yet committed - the commit would then persist a dangling reference.
+ * Requestors may also record the id to have its existence validated by the persistence
+ * target before the data referencing it is committed (trusted-id validation).
*/
objectIdRequestor.registerSkippedOptional(objectId, object, optionalHandler);
}
diff --git a/persistence/persistence/src/main/java/org/eclipse/serializer/persistence/types/PersistenceStoreHandler.java b/persistence/persistence/src/main/java/org/eclipse/serializer/persistence/types/PersistenceStoreHandler.java
index db7a0b88..23c5e711 100644
--- a/persistence/persistence/src/main/java/org/eclipse/serializer/persistence/types/PersistenceStoreHandler.java
+++ b/persistence/persistence/src/main/java/org/eclipse/serializer/persistence/types/PersistenceStoreHandler.java
@@ -82,6 +82,19 @@ public interface PersistenceStoreHandler extends PersistenceFunction, Storer
@Override
public void registerCommitListener(PersistenceCommitListener listener);
+ /**
+ * Reports an object id that this handler writes into the data as a reference without the referenced
+ * instance being available to be stored in the same commit (e.g. an unloaded {@link org.eclipse.serializer.reference.Lazy}
+ * reference's cached id). Implementations may record the id to have its existence validated by the
+ * persistence target before the data referencing it is committed.
+ *
+ * @param objectId the referenced object id that is trusted to already exist in the target.
+ */
+ public default void noteTrustedReference(final long objectId)
+ {
+ // no-op by default
+ }
+
/**
* The retriever used to resolve referenced ids when handlers need to read pre-existing data while
* storing.