gpt2sqlite/sqlite2html.py at main · jbhanks/gpt2sqlite · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
import sys
import sqlite3
import sqlite3
from jinja2 import Template
from pathlib import Path
import re
import html as htmllib  # avoid name conflict
from itertools import cycle

# Nice pastel colors
COLOR_PALETTE = [
    "#e0f7fa",  # cyan
    "#f1f8e9",  # light green
    "#fce4ec",  # pink
    "#fff3e0",  # orange
    "#ede7f6",  # lavender
    "#e8f5e9",  # pale green
    "#f3e5f5",  # light purple
    "#e1f5fe",  # light blue
    "#f9fbe7",  # lime
    "#efebe9",  # stone
]

model_slug_colors = {}


dbpath = sys.argv[1]
outpath = sys.argv[2]

# Output folder
output_dir = Path(outpath)
output_dir.mkdir(exist_ok=True)


# -- Configuration --
DB_PATH = sys.argv[1]
OUTPUT_DIR = Path(sys.argv[2])


# -- Ensure output directory exists --
OUTPUT_DIR.mkdir(exist_ok=True)

# -- Code block converter --
def convert_code_blocks(text):
    if text is None:
        return None

    # Convert triple backticks to <pre><code>
    def repl_block(match):
        code = htmllib.escape(match.group(1))
        return f"<pre><code>{code}</code></pre>"

    text = re.sub(r"```(.*?)```", repl_block, text, flags=re.DOTALL)

    # Convert inline backticks to <code>
    def repl_inline(match):
        code = htmllib.escape(match.group(1))
        return f"<code>{code}</code>"

    text = re.sub(r"`([^`\n]+?)`", repl_inline, text)

    return text

def convert_links(text):
    if text is None:
        return None
    # Replace [title](url) with <a href="url">title</a>
    return re.sub(r'\[([^\]]+)\]\((https?://[^\s)]+)\)', r'<a href="\2">\1</a>', text)

def convert_symbols(text):
    if text is None:
        return None
    # Replace [title](url) with <a href="url">title</a>
    text = re.sub(r'\\cdot', r'·', text)
    # Fix the parenthes/brackets used for equations
    text = re.sub(r'\\\((.*?)\\\)', r'<b>\1</b>', text)
    text = re.sub(r'\\\[(.*?)\\\]', r'<b>\1</b>', text, flags=re.DOTALL)
    return text

# -- HTML template --
html_template = Template("""
<!DOCTYPE html>
<html>
<head>
  <meta charset="UTF-8">
  <title>{{ title }}</title>
  <style>
    body { font-family: sans-serif; line-height: 1.6; }
    .meta { font-size: 0.9em; color: gray; margin-top: 0.25em; }
    hr { margin: 2em 0; }
  </style>
</head>
<body>
  <h1>{{ title }}</h1>
  <h4>Links referenced:</h4>
  <ul>
  {% for link in links %}
  <li>{{link | safe}}</li>
   {% endfor %}
   </ul>
  {% for block in content_blocks %}
    <div style="background-color: {{ block.bgcolor }}; padding: 1em; border-radius: 8px; margin-bottom: 2em;">
    <p><b>{{ block.role }}:</b>
      {{ block.content | safe }}</p>

      <sub>{{ block.create_time }} · {{ block.model_slug | default('', True) }}</sub>
    </div>
    <hr>
  {% endfor %}
</body>
</html>
""")

# -- Database connection --
conn = sqlite3.connect(DB_PATH)
conn.row_factory = sqlite3.Row
cursor = conn.cursor()

# -- Get view names --
cursor.execute("SELECT name FROM sqlite_master WHERE type='view'")
views = [row["name"] for row in cursor.fetchall()]

model_slugs = set()

for view in views:
    try:
        cursor.execute(f"SELECT DISTINCT model_slug FROM {view}")
        model_slugs.update(row["model_slug"] for row in cursor.fetchall() if row["model_slug"] is not None)
    except sqlite3.OperationalError:
        continue

# Assign each model_slug a color from the palette
color_cycle = cycle(COLOR_PALETTE)
for slug in sorted(model_slugs):  # deterministic
    model_slug_colors[slug] = next(color_cycle)

# -- Process each view --
for view in views:
  try:
    cursor.execute(f"SELECT content, role, create_time, model_slug FROM {view}")
    rows = cursor.fetchall()
    content_blocks = []
    for row in rows:
        content = convert_code_blocks(row["content"])
        content = convert_links(content)
        content = convert_symbols(content)
        if content is not None:
          content = re.sub(r'\\\[\n', '[', content)
          content = re.sub(r'\n\\\]', ']', content)
          content = re.sub('\n', '<p>', content)
        if content is not None:
          links = re.findall(r'<a href=.*?</a>', content)
        if content is None:
            continue
        color = model_slug_colors.get(row["model_slug"], "#ffffff")
        content_blocks.append({
            "content": content,
            "role": row["role"],
            "create_time": row["create_time"],
            "model_slug": row["model_slug"],
            "bgcolor": color
        })
    rendered_html = html_template.render(
        title=re.sub('_', ' ', re.sub('chat_[0-9]+_(.*)_?$', r'\1', view).capitalize()),
        content_blocks=content_blocks, links=links
    )

    output_path = OUTPUT_DIR / f"{view}.html"
    output_path.write_text(rendered_html, encoding="utf-8")
  except sqlite3.OperationalError:
    continue
print(f"✅ Exported {len(views)} HTML files to {OUTPUT_DIR.resolve()}")