Skip to content

python: add all seq values as attributes #1239

@milahu

Description

@milahu

useful for autocompletion of attribute names

the python version should add all seq values as attributes
not just the instances values

the default values should be

  • null bytes for known size
  • None for unknown size
  • empty list for repeat types: btree_page.cells = []

... or something like this, to require explicit setting of values

class some_object:
  @property
  def some_attr(self):
    raise Exception("some_attr is unset")

con: this may hurt performance, so this may be optional like

s = kaitaistruct_sqlite3.Sqlite3(init_attrs=True)

example: kaitai_struct_formats/database/sqlite3.ksy

seq:
  - id: magic
    contents: ["SQLite format 3", 0]
  - id: len_page_mod
    type: u2
    doc: |
      The database page size in bytes. Must be a power of two between
      512 and 32768 inclusive, or the value 1 representing a page size
      of 65536.
  - id: write_version
    type: u1
    enum: versions
# ...
  - id: root_page
    type: btree_page
instances:
  len_page:
    value: 'len_page_mod == 1 ? 0x10000 : len_page_mod'
types:
  btree_page:
    seq:
      - id: page_type
        type: u1
      # ...
      - id: cells
        type: ref_cell
        repeat: expr
        repeat-expr: num_cells
import io

# https://github.com/kaitai-io/kaitai_struct_python_runtime
import kaitaistruct

"""
kaitai-struct-compiler --read-write --no-auto-read --target python --import-path kaitai_struct_formats/ kaitai_struct_formats/database/sqlite3.ksy
mv sqlite3.py kaitaistruct_sqlite3.py
"""
import kaitaistruct_sqlite3

num_pages = 1
page_size = 4096

s = kaitaistruct_sqlite3.Sqlite3()

_io = kaitaistruct.KaitaiStream(io.BytesIO(bytearray(num_pages * page_size)))

# print("s attrs:", " ".join(filter(lambda a: not a.startswith("_"), dir(s))))
print("s values:", " ".join(list(filter(lambda a: "a" <= a[0] <= "z", dir(s)))[4:]))

try: print("s.len_page", s.len_page)
except Exception as exc: print(exc)
# AttributeError: 'Sqlite3' object has no attribute 'len_page_mod'

s.len_page_mod = page_size if page_size < 65536 else 1
print("s.len_page", s.len_page)
print("s values:", " ".join(list(filter(lambda a: "a" <= a[0] <= "z", dir(s)))[4:]))

try: s._write(_io)
except Exception as exc: print(exc)
# AttributeError: 'Sqlite3' object has no attribute 'magic'

s.magic = b"SQLite format 3\0"
print("s values:", " ".join(list(filter(lambda a: "a" <= a[0] <= "z", dir(s)))[4:]))

try: s._write(_io)
except Exception as exc: print(exc)
# AttributeError: 'Sqlite3' object has no attribute 'read_version'. Did you mean: 'write_version'?

s.read_version = 0
print("s values:", " ".join(list(filter(lambda a: "a" <= a[0] <= "z", dir(s)))[4:]))

try: s._write(_io)
except Exception as exc: print(exc)
# AttributeError: 'Sqlite3' object has no attribute 'reserved_space'

s.num_pages = num_pages
s.write_version = 0
s.num_freelist_pages = 0
s.schema_format = 4
s.text_encoding = 1 # utf8
s.root_page = s.BtreePage()

print("s values:", " ".join(list(filter(lambda a: "a" <= a[0] <= "z", dir(s)))[4:]))

try: s._write(_io)
except Exception as exc: print(exc)
# 'Sqlite3' object has no attribute 'reserved_space'

output

s values: len_page
'Sqlite3' object has no attribute 'len_page_mod'
s.len_page 4096
s values: len_page len_page_mod
'Sqlite3' object has no attribute 'magic'
s values: len_page len_page_mod magic
'Sqlite3' object has no attribute 'write_version'
s values: len_page len_page_mod magic read_version
'Sqlite3' object has no attribute 'write_version'
s values: len_page len_page_mod magic num_freelist_pages num_pages read_version root_page schema_format text_encoding write_version
'Sqlite3' object has no attribute 'reserved_space'

a more complete test script:

test_kaitai.py
import io

# https://github.com/kaitai-io/kaitai_struct_python_runtime
import kaitaistruct

"""
kaitai-struct-compiler --read-write --no-auto-read --target python --import-path kaitai_struct_formats/ kaitai_struct_formats/database/sqlite3.ksy
mv sqlite3.py kaitaistruct_sqlite3.py
"""
import kaitaistruct_sqlite3
print(f"imported kaitaistruct_sqlite3 from {kaitaistruct_sqlite3.__file__}")

num_pages = 1
page_size = 4096

s = kaitaistruct_sqlite3.Sqlite3()

_io = kaitaistruct.KaitaiStream(io.BytesIO(bytearray(num_pages * page_size)))

def set(on_kn, v):
    on, kn = on_kn.split(".")
    o = eval(on)
    print(f"setting {on}.{kn} = {v!r}")
    setattr(o, kn, v)
    return v

def print_keys(on):
    o = eval(on)
    ignore = ("close", "from_bytes", "from_file", "from_io")
    f = lambda a: "a" <= a[0] <= "z" and a not in ignore
    print(f"{on} keys:", " ".join(filter(f, dir(o))))

def print_value(on_kn):
    on, kn = on_kn.split(".")
    o = eval(on)
    print(f"{on}.{kn} = ", end="")
    try: print(repr(getattr(o, kn)))
    except Exception as exc: print("error:", exc)

def check(on):
    o = eval(on)
    try: print(f"checking {on}: ", end=""); o._check(); print("ok")
    except Exception as exc: print("error:", exc)

def write(on):
    o = eval(on)
    _io.seek(0) # fix: _write__seq does not seek before writing
    try: print(f"writing {on}: ", end=""); o._write(_io); print("ok")
    except Exception as exc: print("error:", exc)

print_keys("s")

print_value("s.len_page")
# AttributeError: 'Sqlite3' object has no attribute 'len_page_mod'

set("s.len_page_mod", page_size if page_size < 65536 else 1)
print_value("s.len_page") # derived from s.len_page_mod
print_keys("s")

write("s")
# AttributeError: 'Sqlite3' object has no attribute 'magic'

set("s.magic", b"SQLite format 3\0")
# set("s.magic", b"some_magic_strr\0") # test
print_keys("s")

write("s")
# AttributeError: 'Sqlite3' object has no attribute 'read_version'. Did you mean: 'write_version'?

set("s.read_version", 0)
print_keys("s")

write("s")
# AttributeError: 'Sqlite3' object has no attribute 'reserved_space'

set("s.num_pages", num_pages)
set("s.write_version", 1)
set("s.read_version", 1)
set("s.reserved_space", 0)
set("s.max_payload_frac", 64)
set("s.min_payload_frac", 32)
set("s.leaf_payload_frac", 32)
set("s.file_change_counter", 0)
# set("s.num_pages", num_pages)
set("s.first_freelist_trunk_page", 0)
set("s.num_freelist_pages", 0)
set("s.schema_cookie", 0)
set("s.schema_format", 4)
set("s.def_page_cache_size", 0)
set("s.largest_root_page", 0)
set("s.text_encoding", 1) # utf8
set("s.user_version", 0)
set("s.is_incremental_vacuum", 0)
set("s.application_id", 0)
set("s.reserved", b"\x00" * 20)
set("s.version_valid_for", 0)
set("s.sqlite_version_number", 0)

check("s")
# 'Sqlite3' object has no attribute 'root_page'

p = set("s.root_page", s.BtreePage())

# FIXME this check should pass
check("s")
# Check failed: root_page, expected: <kaitaistruct_sqlite3.Sqlite3 object at 0x7fc4d624f620>, actual: None

p.page_type = 0x0d # cell_table_leaf
p.first_freeblock = 0
p.num_cells = 0
p.ofs_cells = 0
p.num_frag_free_bytes = 0
# p.right_ptr = 0 # only for (page_type == 2 or page_type == 5)
p.cells = []

print_keys("s")

check("s")
# Check failed: root_page, expected: <kaitaistruct_sqlite3.Sqlite3 object at 0x7fc4d624f620>, actual: None

write("s")
# 'Sqlite3' object has no attribute 'reserved_space'

print("writing done")

_io.seek(0)
_bytes = _io.read_bytes(num_pages * page_size)
# print(_bytes)

o = "test_kaitai.py.db"
print("writing", o)
with open(o, "wb") as f:
    f.write(_bytes)

import sqlite3
con = sqlite3.connect(o)
try: con.execute("select * from sqlite_schema")
except Exception as exc: print(exc)
# sqlite3.DatabaseError: file is not a database

import subprocess
import shlex
import os

o2 = "test_kaitai.py.good.db"

if not os.path.exists(o2):
  args = [
    "sqlite3",
    o2,
    "create table test (id INTEGER)",
  ]
  print(">", shlex.join(args))
  subprocess.run(args)

# TODO rewrite diff in python
args = [
  "diff", "--color=always", "-u",
  "<(", "xxd", o, ")",
  "<(", "xxd", o2, ")",
  "|", "head", "-n20",
]
args = ["bash", "-c", " ".join(args)]
print(">", shlex.join(args))
subprocess.run(args)

Metadata

Metadata

Assignees

No one assigned

    Labels

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions