Add indirection in the directory state tree so that merged nodes would continue sharing state through future merges.

bindings/bindingtester/tests/directory.py

@@ -124,7 +124,7 @@ class DirectoryTest(Test):
             if random.random() < 0.5:
                 while True:
                     self.dir_index = random.randrange(0, len(self.dir_list))
-                    if not self.dir_list[self.dir_index].is_partition or not self.dir_list[self.dir_index].deleted:
+                    if not self.dir_list[self.dir_index].state().is_partition or not self.dir_list[self.dir_index].state().deleted:
                         break
 
                 instructions.push_args(self.dir_index)
@@ -133,15 +133,15 @@ class DirectoryTest(Test):
             dir_entry = self.dir_list[self.dir_index]
 
             choices = op_choices[:]
-            if dir_entry.is_directory:
+            if dir_entry.state().is_directory:
                 choices += directory
-            if dir_entry.is_subspace:
+            if dir_entry.state().is_subspace:
                 choices += subspace
 
             op = random.choice(choices)
 
             # print('%d. Selected %s, dir=%d, dir_id=%d, has_known_prefix=%d, dir_list_len=%d' \
-            #       % (len(instructions), op, self.dir_index, dir_entry.dir_id, dir_entry.has_known_prefix, len(self.dir_list)))
+            #       % (len(instructions), op, self.dir_index, dir_entry.dir_id, dir_entry.state().has_known_prefix, len(self.dir_list)))
 
             if op.endswith('_DATABASE') or op.endswith('_SNAPSHOT'):
                 root_op = op[0:-9]
@@ -194,7 +194,7 @@ class DirectoryTest(Test):
                 if child_entry is None:
                     child_entry = DirectoryStateTreeNode(True, True)
 
-                child_entry.has_known_prefix = False  
+                child_entry.state().has_known_prefix = False  
                 self.dir_list.append(dir_entry.add_child(path, child_entry))
 
             elif root_op == 'DIRECTORY_CREATE':
@@ -224,10 +224,10 @@ class DirectoryTest(Test):
                 if child_entry is None:
                     child_entry = DirectoryStateTreeNode(True, True, has_known_prefix=bool(prefix))
                 elif not bool(prefix):
-                    child_entry.has_known_prefix = False
+                    child_entry.state().has_known_prefix = False
 
                 if is_partition:
-                    child_entry.is_partition = True
+                    child_entry.state().is_partition = True
 
                 self.dir_list.append(dir_entry.add_child(path, child_entry))
 
@@ -256,7 +256,7 @@ class DirectoryTest(Test):
                     self.dir_list.append(dir_entry.add_child(new_path, child_entry))
 
                 # Make sure that the default directory subspace still exists after moving the specified directory
-                if dir_entry.is_directory and not dir_entry.is_subspace and old_path == (u'',):
+                if dir_entry.state().is_directory and not dir_entry.state().is_subspace and old_path == (u'',):
                     self.ensure_default_directory_subspace(instructions, default_path)
 
             elif root_op == 'DIRECTORY_MOVE_TO':
@@ -291,7 +291,7 @@ class DirectoryTest(Test):
                 dir_entry.delete(path)
 
                 # Make sure that the default directory subspace still exists after removing the specified directory
-                if path == () or (dir_entry.is_directory and not dir_entry.is_subspace and path == (u'',)):
+                if path == () or (dir_entry.state().is_directory and not dir_entry.state().is_subspace and path == (u'',)):
                     self.ensure_default_directory_subspace(instructions, default_path)
 
             elif root_op == 'DIRECTORY_LIST' or root_op == 'DIRECTORY_EXISTS':
@@ -310,7 +310,7 @@ class DirectoryTest(Test):
                 instructions.append('DIRECTORY_STRIP_PREFIX')
 
             elif root_op == 'DIRECTORY_UNPACK_KEY' or root_op == 'DIRECTORY_CONTAINS':
-                if not dir_entry.has_known_prefix or random.random() < 0.2 or root_op == 'DIRECTORY_UNPACK_KEY':
+                if not dir_entry.state().has_known_prefix or random.random() < 0.2 or root_op == 'DIRECTORY_UNPACK_KEY':
                     t = self.random.random_tuple(5)
                     instructions.push_args(*test_util.with_length(t))
                     instructions.append('DIRECTORY_PACK_KEY')
@@ -324,7 +324,7 @@ class DirectoryTest(Test):
                 instructions.push_args(*test_util.with_length(t))
                 instructions.append(op)
                 if root_op == 'DIRECTORY_OPEN_SUBSPACE':
-                    self.dir_list.append(DirectoryStateTreeNode(False, True, dir_entry.has_known_prefix))
+                    self.dir_list.append(DirectoryStateTreeNode(False, True, dir_entry.state().has_known_prefix))
                 else:
                     test_util.to_front(instructions, 1)
                     instructions.append('DIRECTORY_STRIP_PREFIX')
@@ -340,10 +340,10 @@ class DirectoryTest(Test):
         for i, dir_entry in enumerate(self.dir_list):
             instructions.push_args(i)
             instructions.append('DIRECTORY_CHANGE')
-            if dir_entry.is_directory:
+            if dir_entry.state().is_directory:
                 instructions.push_args(self.directory_log.key())
                 instructions.append('DIRECTORY_LOG_DIRECTORY')
-            if dir_entry.has_known_prefix and dir_entry.is_subspace:
+            if dir_entry.state().has_known_prefix and dir_entry.state().is_subspace:
                 # print('%d. Logging subspace: %d' % (i, dir_entry.dir_id))
                 instructions.push_args(self.subspace_log.key())
                 instructions.append('DIRECTORY_LOG_SUBSPACE')

bindings/bindingtester/tests/directory_state_tree.py

@@ -1,5 +1,26 @@
 import sys
 
+class TreeNodeState:
+    def __init__(self, dir_id, is_directory, is_subspace, has_known_prefix, root, is_partition):
+        self.dir_id = dir_id
+        self.root = root
+        self.is_directory = is_directory
+        self.is_subspace = is_subspace
+        self.has_known_prefix = has_known_prefix
+        self.children = {}
+        self.deleted = False
+        self.is_partition = is_partition
+
+class Pointer:
+    def __init__(self, obj):
+        self.obj = obj
+
+    def get(self):
+        return self.obj
+
+    def set(self, obj):
+        self.obj = obj
+
 # Represents an element of the directory hierarchy, which could have multiple states 
 class DirectoryStateTreeNode:
     # A cache of directory layers. We mustn't have multiple entries for the same layer
@@ -30,19 +51,15 @@ class DirectoryStateTreeNode:
         return DirectoryStateTreeNode.layers[node_subspace_prefix]
 
     def __init__(self, is_directory, is_subspace, has_known_prefix=True, root=None, is_partition=False):
-        self.root = root or self
+        self.ptr = Pointer(TreeNodeState(DirectoryStateTreeNode.dir_id + 1, is_directory, is_subspace, 
-        self.is_directory = is_directory
+                                         has_known_prefix, root or self, is_partition))
-        self.is_subspace = is_subspace
-        self.has_known_prefix = has_known_prefix
-        self.children = {}
-        self.deleted = False
-        self.is_partition = is_partition
-
-        self.dir_id = DirectoryStateTreeNode.dir_id + 1
         DirectoryStateTreeNode.dir_id += 1
 
+    def state(self):
+        return self.ptr.get()
+
     def __repr__(self):
-        return '{DirEntry %d: %d}' % (self.dir_id, self.has_known_prefix)
+        return '{DirEntry %d: %d}' % (self.state().dir_id, self.state().has_known_prefix)
 
     def _get_descendent(self, subpath, default):
         if not subpath:
@@ -52,9 +69,9 @@ class DirectoryStateTreeNode:
 
         default_child = None
         if default is not None:
-            default_child = default.children.get(subpath[0])
+            default_child = default.state().children.get(subpath[0])
 
-        self_child = self.children.get(subpath[0]) 
+        self_child = self.state().children.get(subpath[0]) 
 
         if self_child is None:
             if default_child is None:
@@ -68,7 +85,7 @@ class DirectoryStateTreeNode:
         return self._get_descendent(subpath, DirectoryStateTreeNode.default_directory)
 
     def add_child(self, subpath, child):
-        child.root = self.root
+        child.state().root = self.state().root
         if DirectoryStateTreeNode.default_directory:
             # print('Adding child %r to default directory at %r' % (child, subpath))
             child = DirectoryStateTreeNode.default_directory._add_child_impl(subpath, child)
@@ -81,56 +98,52 @@ class DirectoryStateTreeNode:
         return c
 
     def _add_child_impl(self, subpath, child):
-        # print('%d, %d. Adding child (recursive): %r' % (self.dir_id, child.dir_id, subpath))
+        # print('%d, %d. Adding child %r (recursive): %r' % (self.state().dir_id, child.state().dir_id, child, subpath))
         if len(subpath) == 0:
-            # print('%d, %d. Setting child: %d, %d' % (self.dir_id, child.dir_id, self.has_known_prefix, child.has_known_prefix))
+            # print('%d, %d. Setting child: %d, %d' % (self.state().dir_id, child.state().dir_id, self.state().has_known_prefix, child.state().has_known_prefix))
             self._merge(child)
             return self
         else:
-            if not subpath[0] in self.children:
+            if not subpath[0] in self.state().children:
-                # print('%d, %d. Path %r was absent from %r (%r)' % (self.dir_id, child.dir_id, subpath[0:1], self, self.children))
+                # print('%d, %d. Path %r was absent from %r (%r)' % (self.state().dir_id, child.state().dir_id, subpath[0:1], self, self.state().children))
-                subdir = DirectoryStateTreeNode(True, True, root=self.root)
+                subdir = DirectoryStateTreeNode(True, True, root=self.state().root)
-                self.children[subpath[0]] = subdir
+                self.state().children[subpath[0]] = subdir
             else:
-                subdir = self.children[subpath[0]]
+                subdir = self.state().children[subpath[0]]
-                # print('%d, %d. Path was present' % (self.dir_id, child.dir_id))
+                # print('%d, %d. Path was present' % (self.state().dir_id, child.state().dir_id))
+
+            if len(subpath) > 1:
+                subdir.state().has_known_prefix = False
 
-            subdir.has_known_prefix = subdir.has_known_prefix and len(subpath) == 1 # For the last element in the path, the merge will take care of has_known_prefix
             return subdir._add_child_impl(subpath[1:], child)
 
     def _merge(self, other):
-        if self.dir_id == other.dir_id:
+        if self.state().dir_id == other.state().dir_id:
             return
 
-        self.is_directory = self.is_directory and other.is_directory
-        self.is_subspace = self.is_subspace and other.is_subspace
-        self.has_known_prefix = self.has_known_prefix and other.has_known_prefix
-        self.deleted = self.deleted or other.deleted
-        self.is_partition = self.is_partition or other.is_partition
-
-        other.root = self.root
-        other.is_directory = self.is_directory
-        other.is_subspace = self.is_subspace
-        other.has_known_prefix = self.has_known_prefix
-        other.deleted = self.deleted
-        other.is_partition = self.is_partition
-        other.dir_id = min(other.dir_id, self.dir_id)
         self.dir_id = other.dir_id
+        self.state().dir_id = min(other.state().dir_id, self.state().dir_id)
+        self.state().is_directory = self.state().is_directory and other.state().is_directory
+        self.state().is_subspace = self.state().is_subspace and other.state().is_subspace
+        self.state().has_known_prefix = self.state().has_known_prefix and other.state().has_known_prefix
+        self.state().deleted = self.state().deleted or other.state().deleted
+        self.state().is_partition = self.state().is_partition or other.state().is_partition
 
-        other_children = other.children.copy()
+        other_children = other.state().children.copy()
-        for c in other_children:
+        other.ptr.set(self.state())
-            if c not in self.children:
+        other.ptr = self.ptr
-                self.children[c] = other_children[c]
-
-        other.children = self.children
 
         for c in other_children:
-            self.children[c]._merge(other_children[c])
+            if c not in self.state().children:
+                self.state().children[c] = other_children[c]
+
+        for c in other_children:
+            self.state().children[c]._merge(other_children[c])
 
     def _delete_impl(self):
-        if not self.deleted:
+        if not self.state().deleted:
-            self.deleted = True
+            self.state().deleted = True
-            for c in self.children.values():
+            for c in self.state().children.values():
                 c._delete_impl()
 
     def delete(self, path):
@@ -139,10 +152,10 @@ class DirectoryStateTreeNode:
             child._delete_impl()
     
 def validate_dir(dir, root):
-    if dir.is_directory:
+    if dir.state().is_directory:
-        assert dir.root == root
+        assert dir.state().root == root
     else:
-        assert dir.root == dir
+        assert dir.state().root == dir
 
 def run_test():
     all_entries = []
@@ -177,7 +190,7 @@ def run_test():
 
     # This directory was merged with the default, but both have readable prefixes
     entry = root.get_descendent(('2',))
-    assert entry.has_known_prefix
+    assert entry.state().has_known_prefix
 
     entry = all_entries[-1]
     all_entries.append(entry.add_child(('1',), DirectoryStateTreeNode(True, True, has_known_prefix=True)))
@@ -191,59 +204,59 @@ def run_test():
 
     all_entries.extend(child_entries)
 
-    # This directory has an known prefix
+    # This directory has an unknown prefix
     entry = root.get_descendent(('1', '2'))
-    assert not entry.has_known_prefix
+    assert not entry.state().has_known_prefix
 
     # This directory was default created and should have an unknown prefix
     # It will merge with the default directory's child, which is not a subspace
     entry = root.get_descendent(('1',))
-    assert not entry.has_known_prefix
+    assert not entry.state().has_known_prefix
-    assert not entry.is_subspace
+    assert not entry.state().is_subspace
 
     # Multiple merges will have made this prefix unreadable
     entry = root.get_descendent(('2',))
-    assert not entry.has_known_prefix
+    assert not entry.state().has_known_prefix
 
     # Merge with default directory's child that has an unknown prefix
     entry = root.get_descendent(('3',))
-    assert not entry.has_known_prefix
+    assert not entry.state().has_known_prefix
 
     # Merge with default directory's child that has an unknown prefix and merged children
     entry = root.get_descendent(('1', '3'))
-    assert set(entry.children.keys()) == {'1', '2', '3', '4', '5', '6'}
+    assert set(entry.state().children.keys()) == {'1', '2', '3', '4', '5', '6'}
 
     # This child entry should be the combination of ['default', '3'], ['default', '1', '3'], and ['1', '3']
     entry = entry.get_descendent(('3',))
-    assert not entry.has_known_prefix
+    assert not entry.state().has_known_prefix
-    assert not entry.is_subspace
+    assert not entry.state().is_subspace
 
     # Verify the merge of the children
-    assert not child_entries[0].has_known_prefix
+    assert not child_entries[0].state().has_known_prefix
-    assert not child_entries[0].is_subspace
+    assert not child_entries[0].state().is_subspace
 
-    assert not child_entries[1].has_known_prefix
+    assert not child_entries[1].state().has_known_prefix
-    assert child_entries[1].is_subspace
+    assert child_entries[1].state().is_subspace
 
-    assert not child_entries[2].has_known_prefix
+    assert not child_entries[2].state().has_known_prefix
-    assert not child_entries[2].is_subspace
+    assert not child_entries[2].state().is_subspace
 
-    assert not child_entries[3].has_known_prefix
+    assert not child_entries[3].state().has_known_prefix
-    assert not child_entries[3].is_subspace
+    assert not child_entries[3].state().is_subspace
 
-    assert child_entries[4].has_known_prefix
+    assert child_entries[4].state().has_known_prefix
-    assert not child_entries[4].is_subspace
+    assert not child_entries[4].state().is_subspace
 
-    assert child_entries[5].has_known_prefix
+    assert child_entries[5].state().has_known_prefix
-    assert child_entries[5].is_subspace
+    assert child_entries[5].state().is_subspace
 
     entry = root.add_child(('3',), entry_to_move)
     all_entries.append(entry)
 
     # Test moving an entry
-    assert not entry.has_known_prefix
+    assert not entry.state().has_known_prefix
-    assert not entry.is_subspace
+    assert not entry.state().is_subspace
-    assert entry.children.keys() == ['1']    
+    assert entry.state().children.keys() == ['1']    
 
     for e in all_entries:
         validate_dir(e, root)