flashback do not depend on mem

README.md

@@ -86,7 +86,7 @@ UPDATE `test`.`test3` SET `addtime`='2016-12-10 13:03:22', `data`='中文', `id`
 
 -K, --no-primary-key 对INSERT语句去除主键。可选。
 
--B, --flashback 生成回滚语句。可选。与stop-never或no-primary-key不能同时添加。
+-B, --flashback 生成回滚语句，可解析大文件，不受内存限制。可选。与stop-never或no-primary-key不能同时添加。
 
 **解析范围控制**
 
@@ -191,13 +191,12 @@ INSERT INTO `test`.`tbl`(`addtime`, `id`, `name`) VALUES ('2016-12-10 00:04:33',
 
 ###限制
 * mysql server必须开启，离线模式下不能解析
-* flashback模式，生成的回滚语句不能超过内存大小(有待优化，mysqlbinlog有同样的问题)
-
 
 ###优点（对比mysqlbinlog）
 
 * 纯Python开发，安装与使用都很简单
 * 自带flashback、no-primary-key解析模式，无需再装补丁
+* flashback模式下，更适合闪回[实战](./example/mysql-flashback-priciple-and-practice.md)
 * 解析为标准SQL，方便理解、调试
 * 代码容易改造，可以支持更多个性化解析
 

binlog2sql/binlog2sql.py

@@ -10,7 +10,7 @@ from pymysqlreplication.row_event import (
     DeleteRowsEvent,
 )
 from pymysqlreplication.event import QueryEvent, RotateEvent, FormatDescriptionEvent
-from binlog2sql_util import command_line_args, concat_sql_from_binlogevent, create_unique_file
+from binlog2sql_util import command_line_args, concat_sql_from_binlogevent, create_unique_file, reversed_lines
 
 class Binlog2sql(object):
 
@@ -101,10 +101,9 @@ class Binlog2sql(object):
                     break
             ftmp.close()
             if self.flashback:
-                # doesn't work if you can't fit the whole file in memory.
-                # need to be optimized
-                for line in reversed(open(tmpFile).readlines()):
-                    print line.rstrip()
+                with open(tmpFile) as ftmp:
+                    for line in reversed_lines(ftmp):
+                        print line.rstrip()
         finally:
             os.remove(tmpFile)
         cur.close()

binlog2sql/binlog2sql_util.py

@@ -176,3 +176,24 @@ def generate_sql_pattern(binlogevent, row=None, flashback=False, nopk=False):
             values = map(fix_object, row['after_values'].values()+row['before_values'].values())
 
     return {'template':template, 'values':values}
+
+def reversed_lines(file):
+    "Generate the lines of file in reverse order."
+    part = ''
+    for block in reversed_blocks(file):
+        for c in reversed(block):
+            if c == '\n' and part:
+                yield part[::-1]
+                part = ''
+            part += c
+    if part: yield part[::-1]
+
+def reversed_blocks(file, blocksize=4096):
+    "Generate blocks of file's contents in reverse order."
+    file.seek(0, os.SEEK_END)
+    here = file.tell()
+    while 0 < here:
+        delta = min(blocksize, here)
+        here -= delta
+        file.seek(here, os.SEEK_SET)
+        yield file.read(delta)