In the process of redoing RevisionedDict
Sven Slootweg
11 years ago
parent
4927b5e7a3
commit
1f32cac481
@ -1,133 +1,159 @@
|
|||||||
import string, random
|
# Problems:
|
||||||
|
# - How to handle list diffs? Can't just replace, would still lose data..
|
||||||
|
# - Over-engineering? Python already interns primitives, so no point in storing object references rather than just direct revision maps?
|
||||||
|
# -> Would still need to pre-process dicts and lists before storage, and compare them...
|
||||||
|
|
||||||
|
# Ideas:
|
||||||
|
# - Download PDF/EPUB headers and extract metadata from there
|
||||||
|
|
||||||
def random_id():
|
import string, random, copy
|
||||||
return "".join(random.choice(string.lowercase + string.uppercase + string.digits) for x in xrange(0, 14))
|
from collections import OrderedDict
|
||||||
|
from difflib import SequenceMatcher
|
||||||
|
|
||||||
class RevisionedDict(object):
|
class RevisionedDict(object):
|
||||||
def __init__(self, parent=None):
|
def __init__(self, data):
|
||||||
self.latest_revision = ""
|
self._revisions = OrderedDict({})
|
||||||
self.parent = parent
|
self._applied_diffs = []
|
||||||
self.revisions = {}
|
self._add_revision(data)
|
||||||
self.objects = {}
|
|
||||||
|
|
||||||
def __eq__(self, other):
|
def _generate_revision_id(self):
|
||||||
# This is a tricky one... we need to compare this RevisionedDict against the other thing - which is almost certainly a dict.
|
return "".join(random.choice(string.lowercase + string.uppercase + string.digits) for x in xrange(0, 14))
|
||||||
# We'll just compare keys and values.
|
|
||||||
|
def _add_revision(self, data):
|
||||||
|
new_revision_id = self._generate_revision_id()
|
||||||
|
self._revisions[new_revision_id] = copy.deepcopy(data)
|
||||||
|
self._last_revision = new_revision_id
|
||||||
|
return new_revision_id
|
||||||
|
|
||||||
|
def _get_last_revision(self): # Always returns a copy!
|
||||||
|
base_revision = copy.deepcopy(self._revisions[self._last_revision])
|
||||||
|
base_revision["_rev"] = self._last_revision # This is to be able to identify the source revision for a modified serialized object later
|
||||||
|
return base_revision
|
||||||
|
|
||||||
|
def _apply_diff(self, diff):
|
||||||
|
new_data = diff.apply(self._get_last_revision(), self._diffs_since(diff.origin_revision))
|
||||||
|
new_revision_id = self._add_revision(new_data)
|
||||||
|
self._applied_diffs.append((new_revision_id, diff))
|
||||||
|
|
||||||
|
def _diffs_since(self, revision_id):
|
||||||
try:
|
try:
|
||||||
if set(self.keys()) != set(other.keys()):
|
revision_index = next(x for x in enumerate(self._applied_diffs) if x[1][0] == revision_id)
|
||||||
return False
|
return [x[1] for x in self._applied_diffs[revision_index[0] + 1:]]
|
||||||
except AttributeError, e:
|
except StopIteration, e:
|
||||||
return False # Not a dict(-like)
|
return [x[1] for x in self._applied_diffs]
|
||||||
|
|
||||||
|
def update(self, data):
|
||||||
|
diff = self.autodiff(data)
|
||||||
|
self._apply_diff(diff)
|
||||||
|
|
||||||
latest_rev = self._get_latest_revision()
|
def diff(self, data, origin_revision):
|
||||||
for key, value in other.iteritems():
|
# Figure out if any revisions happened in the meantime
|
||||||
if self.objects[latest_rev[key]] != value:
|
return RevisionedDictDiff(data, self._revisions[origin_revision], origin_revision)
|
||||||
return False
|
|
||||||
|
def autodiff(self, data):
|
||||||
|
# Takes the revision number from the data
|
||||||
|
return self.diff(data, data["_rev"])
|
||||||
|
|
||||||
|
class RevisionedDictDiff(object):
|
||||||
|
def __init__(self, data, origin_data, origin_revision):
|
||||||
|
self.origin_revision = origin_revision
|
||||||
|
self._diff_data = self._diff_structure(data, origin_data)
|
||||||
|
|
||||||
|
def _diff_structure(self, structure, origin_structure, structure_key=None):
|
||||||
|
if isinstance(structure, dict):
|
||||||
|
if isinstance(origin_structure, dict):
|
||||||
|
# Compare dicts
|
||||||
|
opcodes = []
|
||||||
|
|
||||||
return True
|
removed_keys = set(origin_structure.keys()) - set(structure.keys())
|
||||||
|
|
||||||
def __len__(self):
|
for key in removed_keys:
|
||||||
return len(self._get_latest_revision())
|
opcodes.append(("delete", key))
|
||||||
|
|
||||||
def __getitem__(self, key):
|
new_keys = set(structure.keys()) - set(origin_structure.keys())
|
||||||
return self.objects[self._get_latest_revision()[key]]
|
|
||||||
|
for key in new_keys:
|
||||||
def __setitem__(self, key, value):
|
if key != "_rev": # Ignore added _rev key
|
||||||
obj = self._dump_latest_revision()
|
opcodes.append(("insert", key, structure[key]))
|
||||||
obj[key] = value
|
|
||||||
self.update(obj)
|
|
||||||
|
|
||||||
def __delitem__(self, key):
|
|
||||||
obj = self._dump_latest_revision()
|
|
||||||
del obj[key]
|
|
||||||
self.update(obj)
|
|
||||||
|
|
||||||
def __contains__(self, key):
|
|
||||||
return (key in self._get_latest_revision())
|
|
||||||
|
|
||||||
def keys(self):
|
|
||||||
return self._get_latest_revision().keys()
|
|
||||||
|
|
||||||
def values(self):
|
|
||||||
return [self.objects[id_] for id_ in self._get_latest_revision().values()]
|
|
||||||
|
|
||||||
def items(self):
|
|
||||||
return [(key, self.objects[id_]) for key, id_ in self._get_latest_revision().items()]
|
|
||||||
|
|
||||||
# The below are awful... this really isn't how iterators are supposed to work
|
|
||||||
|
|
||||||
def iterkeys(self):
|
|
||||||
return iter(self._get_latest_revision().keys())
|
|
||||||
|
|
||||||
def itervalues(self):
|
|
||||||
return iter([self.objects[id_] for id_ in self._get_latest_revision().values()])
|
|
||||||
|
|
||||||
def iteritems(self):
|
|
||||||
return iter([(key, self.objects[id_]) for key, id_ in self._get_latest_revision().items()])
|
|
||||||
|
|
||||||
# TODO: __iter__, __reversed__
|
|
||||||
|
|
||||||
def _add_revision(data):
|
|
||||||
object_map = {}
|
|
||||||
latest_rev = self._get_latest_revision()
|
|
||||||
anything_changed = False
|
|
||||||
|
|
||||||
for key in data.keys():
|
|
||||||
try:
|
|
||||||
try:
|
|
||||||
is_dict = isinstance(self.objects[latest_rev[key]][0], RevisionedDict)
|
|
||||||
except IndexError, e:
|
|
||||||
is_dict = False
|
|
||||||
|
|
||||||
if is_dict:
|
for key, value in structure.iteritems():
|
||||||
unchanged = self.objects[latest_rev[key]][0] == data[key]:
|
if key not in new_keys:
|
||||||
else:
|
if value == origin_structure[key]:
|
||||||
unchanged = self.objects[latest_rev[key]] == data[key]:
|
opcodes.append(("equal", key))
|
||||||
except KeyError, e:
|
else:
|
||||||
# Doesn't exist in last rev, new key
|
if isinstance(value, dict) or isinstance(value, list):
|
||||||
unchanged = False
|
opcodes.append(self._diff_structure(value, origin_structure[key], structure_key=key))
|
||||||
|
else:
|
||||||
|
opcodes.append(("replace", key, value))
|
||||||
|
|
||||||
if unchanged:
|
return ("dict", structure_key, opcodes)
|
||||||
# Leave as it is
|
|
||||||
object_map[key] = latest_rev[key]
|
|
||||||
else:
|
else:
|
||||||
# New data!
|
return ("replace", structure)
|
||||||
if isinstance(data[key], dict): # dict, just need to update values
|
elif isinstance(structure, list):
|
||||||
new_sub_rev = self.objects[latest_rev[key]].update(data[key])
|
if isinstance(origin_structure, list):
|
||||||
self.objects[new_id] = (self.objects[latest_rev[key]], new_sub_rev)
|
# Compare lists (does NOT support nested dictionaries yet!)
|
||||||
else:
|
return ("list", structure_key, SequenceMatcher(a=origin_structure, b=structure, autojunk=False).get_opcodes())
|
||||||
new_id = random_id()
|
else:
|
||||||
self.objects[new_id] = data[key]
|
return ("replace", structure)
|
||||||
object_map[key] = new_id
|
|
||||||
anything_changed = True
|
|
||||||
|
|
||||||
if anything_changed:
|
|
||||||
new_rev = random_id()
|
|
||||||
self.revisions[new_rev] = (self.latest_revision, object_map) # (parent revision, new object map)
|
|
||||||
return new_rev
|
|
||||||
else:
|
else:
|
||||||
return latest_rev
|
return ("replace", structure)
|
||||||
|
|
||||||
def _get_latest_revision():
|
def _apply_structure(self, structure, diff_data, intermediate_diffs):
|
||||||
return self.revisions[self.latest_revision]
|
pass
|
||||||
|
# for every key
|
||||||
def _dump_latest_revision():
|
# if list
|
||||||
obj = {}
|
# calculate_offsets (TODO)
|
||||||
for key, id_ in self._get_latest_revision().iteritems():
|
# apply structure
|
||||||
obj[key] = self.objects[id_]
|
# if dict
|
||||||
return obj
|
# apply structure
|
||||||
|
# else
|
||||||
def update(data):
|
# apply diff data
|
||||||
rev_id = self._add_revision(data)
|
# return key
|
||||||
self.latest_revision = rev_id
|
|
||||||
return rev_id
|
def apply(data, intermediate_diffs=[]):
|
||||||
|
# This will apply the diff against the specified source data
|
||||||
# TODO: compare!
|
data = copy.deepcopy(data)
|
||||||
|
self._apply_structure(data, self._diff_data, intermediate_diffs)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
origin = {
|
||||||
|
"type": "message",
|
||||||
|
"data": {
|
||||||
|
"title": "Sample title",
|
||||||
|
"author": "Sample author",
|
||||||
|
"isbn": ["a0", "a1", "a2", "a3"],
|
||||||
|
"description": ["test one", "test two"],
|
||||||
|
"eq": ["a", "b", "c"]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
# Problems:
|
"""
|
||||||
# - How to handle list diffs? Can't just replace, would still lose data..
|
revdict = RevisionedDict(origin)
|
||||||
# - Over-engineering? Python already interns primitives, so no point in storing object references rather than just direct revision maps?
|
origin = revdict._get_last_revision()
|
||||||
# -> Would still need to pre-process dicts and lists before storage, and compare them...
|
|
||||||
|
|
||||||
# Ideas:
|
origin["herp"] = "derp"
|
||||||
# - Download PDF/EPUB headers and extract metadata from there
|
origin["data"]["isbn"].remove("a2")
|
||||||
|
origin["data"]["isbn"].insert(0, "a4")
|
||||||
|
origin["data"]["author"] = "Other author"
|
||||||
|
|
||||||
|
#import json
|
||||||
|
#print json.dumps(revdict.autodiff(origin)._diff_data, indent=4)
|
||||||
|
|
||||||
|
revdict.update(origin)
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
revdict = RevisionedDict(origin)
|
||||||
|
|
||||||
|
for i in xrange(0, 5):
|
||||||
|
x = revdict._add_revision("blah")
|
||||||
|
revdict._applied_diffs.append((x, i))
|
||||||
|
|
||||||
|
base_rev = revdict._last_revision
|
||||||
|
|
||||||
|
for i in xrange(5, 10):
|
||||||
|
x = revdict._add_revision("blah")
|
||||||
|
revdict._applied_diffs.append((x, i))
|
||||||
|
|
||||||
|
print revdict._diffs_since(base_rev)
|
||||||
|
|||||||
Loading…
Reference in New Issue