[Neo-report] r2015 vincent - in /trunk/neo: client/ client/handlers/ tests/client/
nobody at svn.erp5.org
nobody at svn.erp5.org
Fri Apr 23 11:32:22 CEST 2010
Author: vincent
Date: Fri Apr 23 11:32:22 2010
New Revision: 2015
Log:
Fix conflict resolution race-condition.
Fixes the following scenario (committing an object):
2 storage nodes: S1 and S2
1 replicas
- store in S1 (1)
- store in S2 (2)
- S1 answers (1) with a conflict
- client resolves the conflict, and sends to S1 (3) and S2 (4)
- S1 answers (3) with ok
- S2 answers (2) with a conflict
Old code raises because object store counter was not 0 on an object for
which a conflict was reported.
New code ignores answers notifying of a conflict if given serial is the
already-resolved conflict.
Split & extend existing tests.
Modified:
trunk/neo/client/app.py
trunk/neo/client/handlers/storage.py
trunk/neo/tests/client/testStorageHandler.py
Modified: trunk/neo/client/app.py
==============================================================================
--- trunk/neo/client/app.py [iso-8859-1] (original)
+++ trunk/neo/client/app.py [iso-8859-1] Fri Apr 23 11:32:22 2010
@@ -103,6 +103,7 @@
'object_serial_dict': {},
'object_stored_counter_dict': {},
'conflict_serial_dict': {},
+ 'resolved_conflict_serial_dict': {},
'object_stored': 0,
'txn_voted': False,
'txn_finished': False,
@@ -606,8 +607,9 @@
new_data = tryToResolveConflict(oid, conflict_serial, serial,
data)
if new_data is not None:
- # Forget this conflict
- del local_var.conflict_serial_dict[oid]
+ # Mark this conflict as resolved
+ local_var.resolved_conflict_serial_dict = \
+ local_var.conflict_serial_dict.pop(oid)
# Try to store again
self.store(oid, conflict_serial, new_data, version,
local_var.txn)
Modified: trunk/neo/client/handlers/storage.py
==============================================================================
--- trunk/neo/client/handlers/storage.py [iso-8859-1] (original)
+++ trunk/neo/client/handlers/storage.py [iso-8859-1] Fri Apr 23 11:32:22 2010
@@ -71,13 +71,24 @@
local_var = self.app.local_var
object_stored_counter_dict = local_var.object_stored_counter_dict
if conflicting:
- assert object_stored_counter_dict[oid] == 0, \
- object_stored_counter_dict[oid]
- previous_conflict_serial = local_var.conflict_serial_dict.get(oid,
- None)
- assert previous_conflict_serial in (None, serial), \
- (previous_conflict_serial, serial)
- local_var.conflict_serial_dict[oid] = serial
+ conflict_serial_dict = local_var.conflict_serial_dict
+ pending_serial = conflict_serial_dict.get(oid)
+ resolved_serial = local_var.resolved_conflict_serial_dict.get(oid)
+ if pending_serial not in (None, serial) or \
+ resolved_serial not in (None, serial):
+ raise NEOStorageError, 'Multiple conflicts for a single ' \
+ 'object in a single store: %r, %r, %r' % (pending_serial,
+ resolved_serial, serial)
+ # If this conflict is not already resolved, mark it for
+ # resolution.
+ if resolved_serial is None:
+ if object_stored_counter_dict[oid]:
+ raise NEOStorageError, 'Storage node(s) accepted ' \
+ 'object, but one (%s) reports a conflict.' % (
+ dump(conn.getUUID()), )
+ # Note: we might overwrite an entry, but above test protects
+ # against overwriting a different value.
+ conflict_serial_dict[oid] = serial
else:
object_stored_counter_dict[oid] += 1
Modified: trunk/neo/tests/client/testStorageHandler.py
==============================================================================
--- trunk/neo/tests/client/testStorageHandler.py [iso-8859-1] (original)
+++ trunk/neo/tests/client/testStorageHandler.py [iso-8859-1] Fri Apr 23 11:32:22 2010
@@ -87,7 +87,7 @@
self.assertRaises(NEOStorageError, self.handler.answerObject, conn,
*the_object)
- def test_answerStoreObject(self):
+ def test_answerStoreObject_1(self):
conn = self.getConnection()
oid = self.getOID(0)
tid = self.getNextTID()
@@ -95,14 +95,81 @@
local_var = self.app.local_var
local_var.object_stored_counter_dict = {oid: 0}
local_var.conflict_serial_dict = {}
+ local_var.resolved_conflict_serial_dict = {}
self.handler.answerStoreObject(conn, 1, oid, tid)
self.assertEqual(local_var.conflict_serial_dict[oid], tid)
self.assertEqual(local_var.object_stored_counter_dict[oid], 0)
+ self.assertFalse(oid in local_var.resolved_conflict_serial_dict)
+ # object was already accepted by another storage, raise
+ local_var.object_stored_counter_dict = {oid: 1}
+ local_var.conflict_serial_dict = {}
+ local_var.resolved_conflict_serial_dict = {}
+ self.assertRaises(NEOStorageError, self.handler.answerStoreObject,
+ conn, 1, oid, tid)
+
+ def test_answerStoreObject_2(self):
+ conn = self.getConnection()
+ oid = self.getOID(0)
+ tid = self.getNextTID()
+ tid_2 = self.getNextTID()
+ # resolution-pending conflict
+ local_var = self.app.local_var
+ local_var.object_stored_counter_dict = {oid: 0}
+ local_var.conflict_serial_dict = {oid: tid}
+ local_var.resolved_conflict_serial_dict = {}
+ self.handler.answerStoreObject(conn, 1, oid, tid)
+ self.assertEqual(local_var.conflict_serial_dict[oid], tid)
+ self.assertFalse(oid in local_var.resolved_conflict_serial_dict)
+ self.assertEqual(local_var.object_stored_counter_dict[oid], 0)
+ # object was already accepted by another storage, raise
+ local_var.object_stored_counter_dict = {oid: 1}
+ local_var.conflict_serial_dict = {oid: tid}
+ local_var.resolved_conflict_serial_dict = {}
+ self.assertRaises(NEOStorageError, self.handler.answerStoreObject,
+ conn, 1, oid, tid)
+ # detected conflict is different, raise
+ local_var.object_stored_counter_dict = {oid: 0}
+ local_var.conflict_serial_dict = {oid: tid}
+ local_var.resolved_conflict_serial_dict = {}
+ self.assertRaises(NEOStorageError, self.handler.answerStoreObject,
+ conn, 1, oid, tid_2)
+
+ def test_answerStoreObject_3(self):
+ conn = self.getConnection()
+ oid = self.getOID(0)
+ tid = self.getNextTID()
+ tid_2 = self.getNextTID()
+ # already-resolved conflict
+ # This case happens if a storage is answering a store action for which
+ # any other storage already answered (with same conflict) and any other
+ # storage accepted the resolved object.
+ local_var = self.app.local_var
+ local_var.object_stored_counter_dict = {oid: 1}
+ local_var.conflict_serial_dict = {}
+ local_var.resolved_conflict_serial_dict = {oid: tid}
+ self.handler.answerStoreObject(conn, 1, oid, tid)
+ self.assertFalse(oid in local_var.conflict_serial_dict)
+ self.assertEqual(local_var.resolved_conflict_serial_dict[oid], tid)
+ self.assertEqual(local_var.object_stored_counter_dict[oid], 1)
+ # detected conflict is different, raise
+ local_var.object_stored_counter_dict = {oid: 1}
+ local_var.conflict_serial_dict = {}
+ local_var.resolved_conflict_serial_dict = {oid: tid}
+ self.assertRaises(NEOStorageError, self.handler.answerStoreObject,
+ conn, 1, oid, tid_2)
+
+ def test_answerStoreObject_4(self):
+ conn = self.getConnection()
+ oid = self.getOID(0)
+ tid = self.getNextTID()
# no conflict
- local_var.object_stored_counter_dict = {oid: 0}
- local_var.conflict_serial_dict = {}
+ local_var = self.app.local_var
+ local_var.object_stored_counter_dict = {oid: 0}
+ local_var.conflict_serial_dict = {}
+ local_var.resolved_conflict_serial_dict = {}
self.handler.answerStoreObject(conn, 0, oid, tid)
self.assertFalse(oid in local_var.conflict_serial_dict)
+ self.assertFalse(oid in local_var.resolved_conflict_serial_dict)
self.assertEqual(local_var.object_stored_counter_dict[oid], 1)
def test_answerStoreTransaction(self):
More information about the Neo-report
mailing list