-
Notifications
You must be signed in to change notification settings - Fork 202
Open
Labels
Description
useful for autocompletion of attribute names
the python version should add all seq values as attributes
not just the instances values
the default values should be
- null bytes for known size
Nonefor unknown size- empty list for
repeattypes:btree_page.cells = []
... or something like this, to require explicit setting of values
class some_object:
@property
def some_attr(self):
raise Exception("some_attr is unset")con: this may hurt performance, so this may be optional like
s = kaitaistruct_sqlite3.Sqlite3(init_attrs=True)example: kaitai_struct_formats/database/sqlite3.ksy
seq:
- id: magic
contents: ["SQLite format 3", 0]
- id: len_page_mod
type: u2
doc: |
The database page size in bytes. Must be a power of two between
512 and 32768 inclusive, or the value 1 representing a page size
of 65536.
- id: write_version
type: u1
enum: versions
# ...
- id: root_page
type: btree_page
instances:
len_page:
value: 'len_page_mod == 1 ? 0x10000 : len_page_mod'
types:
btree_page:
seq:
- id: page_type
type: u1
# ...
- id: cells
type: ref_cell
repeat: expr
repeat-expr: num_cellsimport io
# https://github.com/kaitai-io/kaitai_struct_python_runtime
import kaitaistruct
"""
kaitai-struct-compiler --read-write --no-auto-read --target python --import-path kaitai_struct_formats/ kaitai_struct_formats/database/sqlite3.ksy
mv sqlite3.py kaitaistruct_sqlite3.py
"""
import kaitaistruct_sqlite3
num_pages = 1
page_size = 4096
s = kaitaistruct_sqlite3.Sqlite3()
_io = kaitaistruct.KaitaiStream(io.BytesIO(bytearray(num_pages * page_size)))
# print("s attrs:", " ".join(filter(lambda a: not a.startswith("_"), dir(s))))
print("s values:", " ".join(list(filter(lambda a: "a" <= a[0] <= "z", dir(s)))[4:]))
try: print("s.len_page", s.len_page)
except Exception as exc: print(exc)
# AttributeError: 'Sqlite3' object has no attribute 'len_page_mod'
s.len_page_mod = page_size if page_size < 65536 else 1
print("s.len_page", s.len_page)
print("s values:", " ".join(list(filter(lambda a: "a" <= a[0] <= "z", dir(s)))[4:]))
try: s._write(_io)
except Exception as exc: print(exc)
# AttributeError: 'Sqlite3' object has no attribute 'magic'
s.magic = b"SQLite format 3\0"
print("s values:", " ".join(list(filter(lambda a: "a" <= a[0] <= "z", dir(s)))[4:]))
try: s._write(_io)
except Exception as exc: print(exc)
# AttributeError: 'Sqlite3' object has no attribute 'read_version'. Did you mean: 'write_version'?
s.read_version = 0
print("s values:", " ".join(list(filter(lambda a: "a" <= a[0] <= "z", dir(s)))[4:]))
try: s._write(_io)
except Exception as exc: print(exc)
# AttributeError: 'Sqlite3' object has no attribute 'reserved_space'
s.num_pages = num_pages
s.write_version = 0
s.num_freelist_pages = 0
s.schema_format = 4
s.text_encoding = 1 # utf8
s.root_page = s.BtreePage()
print("s values:", " ".join(list(filter(lambda a: "a" <= a[0] <= "z", dir(s)))[4:]))
try: s._write(_io)
except Exception as exc: print(exc)
# 'Sqlite3' object has no attribute 'reserved_space'output
s values: len_page
'Sqlite3' object has no attribute 'len_page_mod'
s.len_page 4096
s values: len_page len_page_mod
'Sqlite3' object has no attribute 'magic'
s values: len_page len_page_mod magic
'Sqlite3' object has no attribute 'write_version'
s values: len_page len_page_mod magic read_version
'Sqlite3' object has no attribute 'write_version'
s values: len_page len_page_mod magic num_freelist_pages num_pages read_version root_page schema_format text_encoding write_version
'Sqlite3' object has no attribute 'reserved_space'
a more complete test script:
test_kaitai.py
import io
# https://github.com/kaitai-io/kaitai_struct_python_runtime
import kaitaistruct
"""
kaitai-struct-compiler --read-write --no-auto-read --target python --import-path kaitai_struct_formats/ kaitai_struct_formats/database/sqlite3.ksy
mv sqlite3.py kaitaistruct_sqlite3.py
"""
import kaitaistruct_sqlite3
print(f"imported kaitaistruct_sqlite3 from {kaitaistruct_sqlite3.__file__}")
num_pages = 1
page_size = 4096
s = kaitaistruct_sqlite3.Sqlite3()
_io = kaitaistruct.KaitaiStream(io.BytesIO(bytearray(num_pages * page_size)))
def set(on_kn, v):
on, kn = on_kn.split(".")
o = eval(on)
print(f"setting {on}.{kn} = {v!r}")
setattr(o, kn, v)
return v
def print_keys(on):
o = eval(on)
ignore = ("close", "from_bytes", "from_file", "from_io")
f = lambda a: "a" <= a[0] <= "z" and a not in ignore
print(f"{on} keys:", " ".join(filter(f, dir(o))))
def print_value(on_kn):
on, kn = on_kn.split(".")
o = eval(on)
print(f"{on}.{kn} = ", end="")
try: print(repr(getattr(o, kn)))
except Exception as exc: print("error:", exc)
def check(on):
o = eval(on)
try: print(f"checking {on}: ", end=""); o._check(); print("ok")
except Exception as exc: print("error:", exc)
def write(on):
o = eval(on)
_io.seek(0) # fix: _write__seq does not seek before writing
try: print(f"writing {on}: ", end=""); o._write(_io); print("ok")
except Exception as exc: print("error:", exc)
print_keys("s")
print_value("s.len_page")
# AttributeError: 'Sqlite3' object has no attribute 'len_page_mod'
set("s.len_page_mod", page_size if page_size < 65536 else 1)
print_value("s.len_page") # derived from s.len_page_mod
print_keys("s")
write("s")
# AttributeError: 'Sqlite3' object has no attribute 'magic'
set("s.magic", b"SQLite format 3\0")
# set("s.magic", b"some_magic_strr\0") # test
print_keys("s")
write("s")
# AttributeError: 'Sqlite3' object has no attribute 'read_version'. Did you mean: 'write_version'?
set("s.read_version", 0)
print_keys("s")
write("s")
# AttributeError: 'Sqlite3' object has no attribute 'reserved_space'
set("s.num_pages", num_pages)
set("s.write_version", 1)
set("s.read_version", 1)
set("s.reserved_space", 0)
set("s.max_payload_frac", 64)
set("s.min_payload_frac", 32)
set("s.leaf_payload_frac", 32)
set("s.file_change_counter", 0)
# set("s.num_pages", num_pages)
set("s.first_freelist_trunk_page", 0)
set("s.num_freelist_pages", 0)
set("s.schema_cookie", 0)
set("s.schema_format", 4)
set("s.def_page_cache_size", 0)
set("s.largest_root_page", 0)
set("s.text_encoding", 1) # utf8
set("s.user_version", 0)
set("s.is_incremental_vacuum", 0)
set("s.application_id", 0)
set("s.reserved", b"\x00" * 20)
set("s.version_valid_for", 0)
set("s.sqlite_version_number", 0)
check("s")
# 'Sqlite3' object has no attribute 'root_page'
p = set("s.root_page", s.BtreePage())
# FIXME this check should pass
check("s")
# Check failed: root_page, expected: <kaitaistruct_sqlite3.Sqlite3 object at 0x7fc4d624f620>, actual: None
p.page_type = 0x0d # cell_table_leaf
p.first_freeblock = 0
p.num_cells = 0
p.ofs_cells = 0
p.num_frag_free_bytes = 0
# p.right_ptr = 0 # only for (page_type == 2 or page_type == 5)
p.cells = []
print_keys("s")
check("s")
# Check failed: root_page, expected: <kaitaistruct_sqlite3.Sqlite3 object at 0x7fc4d624f620>, actual: None
write("s")
# 'Sqlite3' object has no attribute 'reserved_space'
print("writing done")
_io.seek(0)
_bytes = _io.read_bytes(num_pages * page_size)
# print(_bytes)
o = "test_kaitai.py.db"
print("writing", o)
with open(o, "wb") as f:
f.write(_bytes)
import sqlite3
con = sqlite3.connect(o)
try: con.execute("select * from sqlite_schema")
except Exception as exc: print(exc)
# sqlite3.DatabaseError: file is not a database
import subprocess
import shlex
import os
o2 = "test_kaitai.py.good.db"
if not os.path.exists(o2):
args = [
"sqlite3",
o2,
"create table test (id INTEGER)",
]
print(">", shlex.join(args))
subprocess.run(args)
# TODO rewrite diff in python
args = [
"diff", "--color=always", "-u",
"<(", "xxd", o, ")",
"<(", "xxd", o2, ")",
"|", "head", "-n20",
]
args = ["bash", "-c", " ".join(args)]
print(">", shlex.join(args))
subprocess.run(args)