From dfbfe2e30963d2dddc915fac4b1ff7981b75ada3 Mon Sep 17 00:00:00 2001
From: Bruno Barros <bruno@barrosnet.com>
Date: Fri, 11 Jul 2025 15:39:56 -0300
Subject: [PATCH] Converting otbm2json to Python with to_json and to_otbm
 example scripts

---
 .gitignore      |   9 ++
 README.md       |  17 ++-
 py/headers.py   |  36 +++++
 py/otbm2json.py | 385 ++++++++++++++++++++++++++++++++++++++++++++++++
 py/to_json.py   |  13 ++
 py/to_otbm.py   |  12 ++
 6 files changed, 469 insertions(+), 3 deletions(-)
 create mode 100644 .gitignore
 create mode 100644 py/headers.py
 create mode 100644 py/otbm2json.py
 create mode 100644 py/to_json.py
 create mode 100644 py/to_otbm.py

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..1f91850
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,9 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# PyInstaller
+dist/
+build/
+*.spec
diff --git a/README.md b/README.md
index 2773880..3a5d53e 100644
--- a/README.md
+++ b/README.md
@@ -1,10 +1,13 @@
 # OTBM2JSON
+
 NodeJS library for programmatically modifying Open Tibia Binary Mapping files. This framework reads .otbm files and parses them to an intermediary JSON format. This JSON structure can be changed programatically to make generic modifications. Once a change has been committed to the structure, it can be encoded back to an .otbm file.
 
 # JSON Structure
+
 The structure of the intermediary JSON format read from and to `.otbm` can be seen in the example `OTBM.json`.
 
 # Usage
+
 Import the library in your script:
 
     const otbm2json = require("./otbm2json.js");
@@ -12,15 +15,16 @@ Import the library in your script:
 The library provides two functions for reading and writing OTBM:
 
     data = otbm2json.read(filename);
-    
+
     ** Modify the data object here **
-    
+
     otbm2json.write(filename, data);
 
 For an example see below.
 
 # Example
-An example script `examples/example.js` is provided. This script uses the `examples/void.otbm` (8x8 void area) in this repository and replaces all void tiles with chessboard tiles and writes the result to  `examples/chess.otbm`.
+
+An example script `examples/example.js` is provided. This script uses the `examples/void.otbm` (8x8 void area) in this repository and replaces all void tiles with chessboard tiles and writes the result to `examples/chess.otbm`.
 
 <p align="center">
   <img src="images/void.png">
@@ -28,5 +32,12 @@ An example script `examples/example.js` is provided. This script uses the `examp
   <img src="images/chess.png">
 </p>
 
+# Python
+
+There is also a Python version of this script in `/py` directory.
+You can run `python ./py/to_json.py` or `python ./py/to_otbm.py` as example.
+This is very handy for handling large files.
+
 # Version
+
 Current version 0.2.0. This is a work in progress.
diff --git a/py/headers.py b/py/headers.py
new file mode 100644
index 0000000..4680dd3
--- /dev/null
+++ b/py/headers.py
@@ -0,0 +1,36 @@
+HEADERS = {
+    "OTBM_MAP_HEADER": 0x00,
+    "OTBM_MAP_DATA": 0x02,
+    "OTBM_TILE_AREA": 0x04,
+    "OTBM_TILE": 0x05,
+    "OTBM_ITEM": 0x06,
+    "OTBM_TOWNS": 0x0C,
+    "OTBM_TOWN": 0x0D,
+    "OTBM_HOUSETILE": 0x0E,
+    "OTBM_WAYPOINTS": 0x0F,
+    "OTBM_WAYPOINT": 0x10,
+
+    "OTBM_ATTR_DESCRIPTION": 0x01,
+    "OTBM_ATTR_EXT_FILE": 0x02,
+    "OTBM_ATTR_TILE_FLAGS": 0x03,
+    "OTBM_ATTR_ACTION_ID": 0x04,
+    "OTBM_ATTR_UNIQUE_ID": 0x05,
+    "OTBM_ATTR_TEXT": 0x06,
+    "OTBM_ATTR_DESC": 0x07,
+    "OTBM_ATTR_TELE_DEST": 0x08,
+    "OTBM_ATTR_ITEM": 0x09,
+    "OTBM_ATTR_DEPOT_ID": 0x0A,
+    "OTBM_ATTR_EXT_SPAWN_FILE": 0x0B,
+    "OTBM_ATTR_EXT_HOUSE_FILE": 0x0D,
+    "OTBM_ATTR_HOUSEDOORID": 0x0E,
+    "OTBM_ATTR_COUNT": 0x0F,
+    "OTBM_ATTR_RUNE_CHARGES": 0x16,
+
+    "TILESTATE_NONE": 0x0000,
+    "TILESTATE_PROTECTIONZONE": 0x0001,
+    "TILESTATE_DEPRECATED": 0x0002,
+    "TILESTATE_NOPVP": 0x0004,
+    "TILESTATE_NOLOGOUT": 0x0008,
+    "TILESTATE_PVPZONE": 0x0010,
+    "TILESTATE_REFRESH": 0x0020,
+}
diff --git a/py/otbm2json.py b/py/otbm2json.py
new file mode 100644
index 0000000..92f56bf
--- /dev/null
+++ b/py/otbm2json.py
@@ -0,0 +1,385 @@
+import struct
+from headers import HEADERS
+
+NODE_ESC = 0xFD
+NODE_INIT = 0xFE
+NODE_TERM = 0xFF
+__VERSION__ = "1.0.1"
+
+
+def write(outfile, data):
+    with open(outfile, "wb") as f:
+        f.write(serialize(data))
+
+
+def serialize(data):
+    # OTBM Header (4 bytes zeros)
+    result = bytearray([0x00, 0x00, 0x00, 0x00])
+    result.extend(write_node(data["data"]))
+    return bytes(result)
+
+
+def write_node(node):
+    return b"".join(
+        [
+            bytes([NODE_INIT]),
+            write_element(node),
+            b"".join(write_node(child) for child in get_child_nodes(node)),
+            bytes([NODE_TERM]),
+        ]
+    )
+
+
+def get_child_nodes(node):
+    t = node.get("type")
+    if t == HEADERS["OTBM_TILE_AREA"]:
+        return node.get("tiles", [])
+    elif t == HEADERS["OTBM_TILE"] or t == HEADERS["OTBM_HOUSETILE"]:
+        return node.get("items", [])
+    elif t == HEADERS["OTBM_TOWNS"]:
+        return node.get("towns", [])
+    elif t == HEADERS["OTBM_ITEM"]:
+        return node.get("content", [])
+    elif t == HEADERS["OTBM_MAP_DATA"]:
+        return node.get("features", [])
+    else:
+        return node.get("nodes", [])
+
+
+def write_element(node):
+    t = node.get("type")
+    if t == HEADERS["OTBM_MAP_HEADER"]:
+        buf = struct.pack(
+            "<B I H H I I",
+            HEADERS["OTBM_MAP_HEADER"],
+            node["version"],
+            node["mapWidth"],
+            node["mapHeight"],
+            node["itemsMajorVersion"],
+            node["itemsMinorVersion"],
+        )
+    elif t == HEADERS["OTBM_MAP_DATA"]:
+        buf = struct.pack("<B", HEADERS["OTBM_MAP_DATA"]) + write_attributes(node)
+    elif t == HEADERS["OTBM_TILE_AREA"]:
+        buf = struct.pack(
+            "<B H H B", HEADERS["OTBM_TILE_AREA"], node["x"], node["y"], node["z"]
+        )
+    elif t == HEADERS["OTBM_TILE"]:
+        buf = struct.pack("<B B B", HEADERS["OTBM_TILE"], node["x"], node["y"])
+        buf += write_attributes(node)
+    elif t == HEADERS["OTBM_HOUSETILE"]:
+        buf = struct.pack(
+            "<B B B I", HEADERS["OTBM_HOUSETILE"], node["x"], node["y"], node["houseId"]
+        )
+        buf += write_attributes(node)
+    elif t == HEADERS["OTBM_ITEM"]:
+        buf = struct.pack("<B H", HEADERS["OTBM_ITEM"], node["id"])
+        buf += write_attributes(node)
+    elif t == HEADERS["OTBM_WAYPOINT"]:
+        name = node["name"].encode("ascii")
+        buf = struct.pack("<B H", HEADERS["OTBM_WAYPOINT"], len(name)) + name
+        buf += struct.pack("<H H B", node["x"], node["y"], node["z"])
+    elif t == HEADERS["OTBM_WAYPOINTS"]:
+        buf = struct.pack("<B", HEADERS["OTBM_WAYPOINTS"])
+    elif t == HEADERS["OTBM_TOWNS"]:
+        buf = struct.pack("<B", HEADERS["OTBM_TOWNS"])
+    elif t == HEADERS["OTBM_TOWN"]:
+        name = node["name"].encode("ascii")
+        buf = (
+            struct.pack("<B I H", HEADERS["OTBM_TOWN"], node["townid"], len(name))
+            + name
+        )
+        buf += struct.pack("<H H B", node["x"], node["y"], node["z"])
+    else:
+        raise Exception("Could not write node. Unknown node type: %s" % str(t))
+    return escape_characters(buf)
+
+
+def escape_characters(buf):
+    # Escapes 0xFD, 0xFE, 0xFF
+    out = bytearray()
+    for b in buf:
+        if b in (NODE_TERM, NODE_INIT, NODE_ESC):
+            out.append(NODE_ESC)
+        out.append(b)
+    return bytes(out)
+
+
+def write_ascii_string_16le(s):
+    if isinstance(s, str):
+        s = s.encode("utf-8")
+    return struct.pack("<H", len(s)) + s
+
+
+def write_attributes(node):
+    attr = bytearray()
+    H = HEADERS
+    if node.get("destination"):
+        d = node["destination"]
+        attr += struct.pack(
+            "<B H H B", H["OTBM_ATTR_TELE_DEST"], d["x"], d["y"], d["z"]
+        )
+    if node.get("description"):
+        attr += struct.pack("<B", H["OTBM_ATTR_DESCRIPTION"])
+        attr += write_ascii_string_16le(node["description"])
+    if node.get("uid"):
+        attr += struct.pack("<B H", H["OTBM_ATTR_UNIQUE_ID"], node["uid"])
+    if node.get("aid"):
+        attr += struct.pack("<B H", H["OTBM_ATTR_ACTION_ID"], node["aid"])
+    if node.get("runeCharges"):
+        attr += struct.pack("<B H", H["OTBM_ATTR_RUNE_CHARGES"], node["runeCharges"])
+    if node.get("spawnfile"):
+        attr += struct.pack("<B", H["OTBM_ATTR_EXT_SPAWN_FILE"])
+        attr += write_ascii_string_16le(node["spawnfile"])
+    if node.get("text"):
+        attr += struct.pack("<B", H["OTBM_ATTR_TEXT"])
+        attr += write_ascii_string_16le(node["text"])
+    if node.get("housefile"):
+        attr += struct.pack("<B", H["OTBM_ATTR_EXT_HOUSE_FILE"])
+        attr += write_ascii_string_16le(node["housefile"])
+    if node.get("tileid"):
+        attr += struct.pack("<B H", H["OTBM_ATTR_ITEM"], node["tileid"])
+    if node.get("count"):
+        attr += struct.pack("<B B", H["OTBM_ATTR_COUNT"], node["count"])
+    if node.get("depotId"):
+        attr += struct.pack("<B H", H["OTBM_ATTR_DEPOT_ID"], node["depotId"])
+    if node.get("houseDoorId"):
+        attr += struct.pack("<B B", H["OTBM_ATTR_HOUSEDOORID"], node["houseDoorId"])
+    if node.get("zones"):
+        attr += struct.pack(
+            "<B I", H["OTBM_ATTR_TILE_FLAGS"], write_flags(node["zones"])
+        )
+    return bytes(attr)
+
+
+def write_flags(zones):
+    H = HEADERS
+    flags = H["TILESTATE_NONE"]
+    if zones.get("protection"):
+        flags |= H["TILESTATE_PROTECTIONZONE"]
+    if zones.get("noPVP"):
+        flags |= H["TILESTATE_NOPVP"]
+    if zones.get("noLogout"):
+        flags |= H["TILESTATE_NOLOGOUT"]
+    if zones.get("PVPZone"):
+        flags |= H["TILESTATE_PVPZONE"]
+    if zones.get("refresh"):
+        flags |= H["TILESTATE_REFRESH"]
+    return flags
+
+
+def read(infile):
+    with open(infile, "rb") as f:
+        data = f.read()
+    map_identifier = struct.unpack("<I", data[:4])[0]
+    if map_identifier not in (0x00000000, 0x4D42544F):
+        raise Exception("Unknown OTBM format: unexpected magic bytes.")
+    node, _ = read_node(data[4:])
+    return {
+        "version": __VERSION__,
+        "identifier": map_identifier,
+        "data": node,
+    }
+
+
+def read_node(data):
+    assert data[0] == NODE_INIT
+    i = 1  # Skip NODE_INIT itself
+    node_data_end = None
+    # 1. Finds the end of node_data (before children)
+    while i < len(data):
+        c_byte = data[i]
+        if c_byte == NODE_ESC:
+            i += 2  # skip ESC + byte
+            continue
+        if c_byte == NODE_INIT or c_byte == NODE_TERM:
+            node_data_end = i
+            break
+        i += 1
+    if node_data_end is None:
+        raise Exception("Malformed OTBM node: no end for node_data")
+    node_data = data[1:node_data_end]
+    i = node_data_end
+    children = []
+    # 2. Read children
+    while i < len(data):
+        c_byte = data[i]
+        if c_byte == NODE_ESC:
+            i += 2
+            continue
+        if c_byte == NODE_INIT:
+            child, child_len = read_node(data[i:])
+            children.append(child)
+            i += child_len
+            continue
+        if c_byte == NODE_TERM:
+            node, _ = parse_node(node_data, children)
+            return node, i + 1
+        i += 1
+    raise Exception("Malformed OTBM node: no closing NODE_TERM")
+
+
+def parse_node(data, children):
+    H = HEADERS
+    if not data:
+        return {"type": None}, 0
+    node_type = data[0]
+    node = {}
+    if node_type == H["OTBM_MAP_HEADER"]:
+        (version, mapWidth, mapHeight, itemsMajorVersion, itemsMinorVersion) = (
+            struct.unpack("<I H H I I", data[1:17])
+        )
+        node = {
+            "type": H["OTBM_MAP_HEADER"],
+            "version": version,
+            "mapWidth": mapWidth,
+            "mapHeight": mapHeight,
+            "itemsMajorVersion": itemsMajorVersion,
+            "itemsMinorVersion": itemsMinorVersion,
+        }
+    elif node_type == H["OTBM_MAP_DATA"]:
+        node = {"type": H["OTBM_MAP_DATA"]}
+        node.update(read_attributes(data[1:]))
+    elif node_type == H["OTBM_TILE_AREA"]:
+        x, y, z = struct.unpack("<H H B", data[1:6])
+        node = {"type": H["OTBM_TILE_AREA"], "x": x, "y": y, "z": z}
+    elif node_type == H["OTBM_TILE"]:
+        x, y = struct.unpack("<B B", data[1:3])
+        node = {"type": H["OTBM_TILE"], "x": x, "y": y}
+        node.update(read_attributes(data[3:]))
+    elif node_type == H["OTBM_HOUSETILE"]:
+        x, y, houseId = struct.unpack("<B B I", data[1:7])
+        node = {"type": H["OTBM_HOUSETILE"], "x": x, "y": y, "houseId": houseId}
+        node.update(read_attributes(data[7:]))
+    elif node_type == H["OTBM_ITEM"]:
+        id = struct.unpack("<H", data[1:3])[0]
+        node = {"type": H["OTBM_ITEM"], "id": id}
+        node.update(read_attributes(data[3:]))
+    elif node_type == H["OTBM_WAYPOINT"]:
+        name_len = struct.unpack("<H", data[1:3])[0]
+        name = data[3 : 3 + name_len].decode("ascii")
+        x, y, z = struct.unpack("<H H B", data[3 + name_len : 3 + name_len + 5])
+        node = {"type": H["OTBM_WAYPOINT"], "name": name, "x": x, "y": y, "z": z}
+    elif node_type == H["OTBM_WAYPOINTS"]:
+        node = {"type": H["OTBM_WAYPOINTS"]}
+    elif node_type == H["OTBM_TOWNS"]:
+        node = {"type": H["OTBM_TOWNS"]}
+    elif node_type == H["OTBM_TOWN"]:
+        townid = struct.unpack("<I", data[1:5])[0]
+        name_len = struct.unpack("<H", data[5:7])[0]
+        name = data[7 : 7 + name_len].decode("ascii")
+        x, y, z = struct.unpack("<H H B", data[7 + name_len : 7 + name_len + 5])
+        node = {
+            "type": H["OTBM_TOWN"],
+            "townid": townid,
+            "name": name,
+            "x": x,
+            "y": y,
+            "z": z,
+        }
+    # Set children according to node type
+    if node_type == H["OTBM_TILE_AREA"]:
+        if children:
+            node["tiles"] = children
+    elif node_type in (H["OTBM_TILE"], H["OTBM_HOUSETILE"]):
+        if children:
+            node["items"] = children
+    elif node_type == H["OTBM_TOWNS"]:
+        if children:
+            node["towns"] = children
+    elif node_type == H["OTBM_ITEM"]:
+        if children:
+            node["content"] = children
+    elif node_type == H["OTBM_MAP_DATA"]:
+        if children:
+            node["features"] = children
+    else:
+        if children:
+            node["nodes"] = children
+    return node, len(data) + 1
+
+
+def read_ascii_string_16le(data):
+    length = struct.unpack("<H", data[:2])[0]
+    return data[2 : 2 + length].decode("ascii"), length + 2
+
+
+def read_attributes(data):
+    H = HEADERS
+    i = 0
+    properties = {}
+    while i + 1 < len(data):
+        attr_type = data[i]
+        i += 1
+        if attr_type == H["OTBM_ATTR_TEXT"]:
+            s, l = read_ascii_string_16le(data[i:])
+            properties["text"] = s
+            i += l
+        elif attr_type == H["OTBM_ATTR_EXT_SPAWN_FILE"]:
+            s, l = read_ascii_string_16le(data[i:])
+            properties["spawnfile"] = s
+            i += l
+        elif attr_type == H["OTBM_ATTR_EXT_HOUSE_FILE"]:
+            s, l = read_ascii_string_16le(data[i:])
+            properties["housefile"] = s
+            i += l
+        elif attr_type == H["OTBM_ATTR_HOUSEDOORID"]:
+            properties["houseDoorId"] = data[i]
+            i += 1
+        elif attr_type == H["OTBM_ATTR_DESCRIPTION"]:
+            s, l = read_ascii_string_16le(data[i:])
+            if "description" in properties:
+                properties["description"] += " " + s
+            else:
+                properties["description"] = s
+            i += l
+        elif attr_type == H["OTBM_ATTR_DESC"]:
+            s, l = read_ascii_string_16le(data[i:])
+            properties["text"] = s
+            i += l
+        elif attr_type == H["OTBM_ATTR_DEPOT_ID"]:
+            properties["depotId"] = struct.unpack("<H", data[i : i + 2])[0]
+            i += 2
+        elif attr_type == H["OTBM_ATTR_TILE_FLAGS"]:
+            properties["zones"] = read_flags(struct.unpack("<I", data[i : i + 4])[0])
+            i += 4
+        elif attr_type == H["OTBM_ATTR_RUNE_CHARGES"]:
+            properties["runeCharges"] = struct.unpack("<H", data[i : i + 2])[0]
+            i += 2
+        elif attr_type == H["OTBM_ATTR_COUNT"]:
+            properties["count"] = data[i]
+            i += 1
+        elif attr_type == H["OTBM_ATTR_ITEM"]:
+            properties["tileid"] = struct.unpack("<H", data[i : i + 2])[0]
+            i += 2
+        elif attr_type == H["OTBM_ATTR_ACTION_ID"]:
+            properties["aid"] = struct.unpack("<H", data[i : i + 2])[0]
+            i += 2
+        elif attr_type == H["OTBM_ATTR_UNIQUE_ID"]:
+            properties["uid"] = struct.unpack("<H", data[i : i + 2])[0]
+            i += 2
+        elif attr_type == H["OTBM_ATTR_TELE_DEST"]:
+            x = struct.unpack("<H", data[i : i + 2])[0]
+            y = struct.unpack("<H", data[i + 2 : i + 4])[0]
+            z = data[i + 4]
+            properties["destination"] = {"x": x, "y": y, "z": z}
+            i += 5
+        else:
+            # Unknown attribute, skip
+            break
+    return properties
+
+
+def read_flags(flags):
+    H = HEADERS
+    return {
+        "protection": int(bool(flags & H["TILESTATE_PROTECTIONZONE"])),
+        "noPVP": int(bool(flags & H["TILESTATE_NOPVP"])),
+        "noLogout": int(bool(flags & H["TILESTATE_NOLOGOUT"])),
+        "PVPZone": int(bool(flags & H["TILESTATE_PVPZONE"])),
+        "refresh": int(bool(flags & H["TILESTATE_REFRESH"])),
+    }
+
+
+# Expose the main functions
+__all__ = ["read", "write", "serialize", "__VERSION__", "HEADERS"]
diff --git a/py/to_json.py b/py/to_json.py
new file mode 100644
index 0000000..d60a0e8
--- /dev/null
+++ b/py/to_json.py
@@ -0,0 +1,13 @@
+import otbm2json
+import orjson
+import time
+
+start = time.time()
+
+obj = otbm2json.read("map.otbm")
+with open("OTBM.json", "wb") as f:
+    f.write(orjson.dumps(obj, option=orjson.OPT_NON_STR_KEYS))
+    f.write(b"\n")
+
+end = time.time()
+print(f"Total execution time: {end - start:.2f} seconds.")
diff --git a/py/to_otbm.py b/py/to_otbm.py
new file mode 100644
index 0000000..4d23abc
--- /dev/null
+++ b/py/to_otbm.py
@@ -0,0 +1,12 @@
+import otbm2json
+import json
+import time
+
+start = time.time()
+
+with open("OTBM.json") as f:
+    obj = json.load(f)
+    otbm2json.write("map.otbm", obj)
+
+end = time.time()
+print(f"Total execution time: {end - start:.2f} seconds.")