summaryrefslogtreecommitdiffstats
path: root/utils/nwztools/database/gen_db.py
blob: de78d05cabe23247b8a46c914077dd7d6a39c81e (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
#!/usr/bin/python3
import glob
import os
import re
import subprocess
import hashlib

# parse models.txt
g_models = []
with open('models.txt') as fp:
    for line in fp:
        # we unpack and repack 1) to make the format obvious 2) to catch errors
        mid,name = line.rstrip().split(",")
        g_models.append({'mid': int(mid, 0), 'name': name})
# parse series.txt
g_series = []
g_series_codename = set()
with open('series.txt') as fp:
    for line in fp:
        # we unpack and repack 1) to make the format obvious 2) to catch errors
        arr = line.rstrip().split(",")
        codename = arr[0]
        name = arr[1]
        models = arr[2:]
        # handle empty list
        if len(models) == 1 and models[0] == "":
            models = []
        models = [int(mid,0) for mid in models]
        g_series.append({'codename': codename, 'name': name, 'models': models})
        g_series_codename.add(codename)
# parse all maps in nvp/
# since most nvps are the same, what we actually do is to compute the md5sum hash
# of all files, to identify groups and then each entry in the name is in fact the
# hash, and we only parse one file per hash group
g_hash_nvp = dict() # hash -> nvp
g_nvp_hash = dict() # codename -> hash
HASH_SIZE=6
map_files = glob.glob('nvp/nw*.txt')
for f in map_files:
    h = hashlib.md5()
    h.update(open(f, "rb").read())
    hash = h.hexdigest()
    codename = re.search('(nw.*)\.txt', f).group(1)
    # sanity check
    if not (codename in g_series_codename):
        print("Warning: file %s does not have a match series in series.txt" % f)
    hash = hash[:HASH_SIZE]
    # only keep one file
    if not (hash in g_hash_nvp):
        g_hash_nvp[hash] = set()
    g_hash_nvp[hash].add(codename);
    g_nvp_hash[codename] = hash
# we have some file nodes (nodes-*) but not necessarily for all series
# so for each hash group, try to find at least one
for hash in g_hash_nvp:
    # look at all codename and see if we can find one with a node file
    node_codename = ""
    for codename in g_hash_nvp[hash]:
        if os.path.isfile("nvp/nodes-%s.txt" % codename):
            node_codename = codename
            break
    # if we didn't find one, we just keep the first one
    # otherwise keep the one we found
    if node_codename == "":
        node_codename = g_hash_nvp[hash].pop()
    g_hash_nvp[hash] = node_codename
# for each entry in g_hash_nvp, replace the file name by the actual table
# that we parse, and compute all nvp names at the same time
g_nvp_names = set() # set of all nvp names
g_nvp_desc = dict() # name -> set of all description of a node
g_nvp_size = dict() # name -> set of all possible sizes of a node
for hash in g_hash_nvp:
    codename = g_hash_nvp[hash]
    # extract codename from file
    # parse file
    map = dict()
    with open("nvp/%s.txt" % codename) as fp:
        for line in fp:
            # we unpack and repack 1) to make the format obvious 2) to catch errors
            name,index = line.rstrip().split(",")
            # convert node to integer but be careful of leading 0 (ie 010 is actually
            # 10 in decimal, it is not in octal)
            index = int(index, 10)
            map[index] = name
            g_nvp_names.add(name)
    # parse node map if any
    node_map = dict()
    if os.path.isfile("nvp/nodes-%s.txt" % codename):
        with open("nvp/nodes-%s.txt" % codename) as fp:
            for line in fp:
                # we unpack and repack 1) to make the format obvious 2) to catch errors
                index,size,desc = line.rstrip().split(",")
                # convert node to integer but be careful of leading 0 (ie 010 is actually
                # 10 in decimal, it is not in octal)
                index = int(index, 10)
                desc = desc.rstrip()
                node_map[index] = {'size': size, 'desc': desc}
    # compute final nvp
    nvp = dict()
    for index in map:
        size = 0
        desc = ""
        name = map[index]
        if index in node_map:
            size = node_map[index]["size"]
            desc = node_map[index]["desc"]
        nvp[name] = index
        if not (name in g_nvp_desc):
            g_nvp_desc[name] = set()
        if len(desc) != 0:
            g_nvp_desc[name].add(desc)
        if not (name in g_nvp_size):
            g_nvp_size[name] = set()
        if size != 0:
            g_nvp_size[name].add(size)
    g_hash_nvp[hash] = nvp

#
# generate header
#
header_begin = \
"""\
/***************************************************************************
 *             __________               __   ___.
 *   Open      \______   \ ____   ____ |  | _\_ |__   _______  ___
 *   Source     |       _//  _ \_/ ___\|  |/ /| __ \ /  _ \  \/  /
 *   Jukebox    |    |   (  <_> )  \___|    < | \_\ (  <_> > <  <
 *   Firmware   |____|_  /\____/ \___  >__|_ \|___  /\____/__/\_ \\
 *                     \/            \/     \/    \/            \/
 *
 * Copyright (C) 2016 Amaury Pouly
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version 2
 * of the License, or (at your option) any later version.
 *
 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
 * KIND, either express or implied.
 *
 ****************************************************************************/
#ifndef __NWZ_DB_H__
#define __NWZ_DB_H__

/** /!\ This file was automatically generated, DO NOT MODIFY IT DIRECTLY /!\ */

/* List of all known NVP nodes */
enum nwz_nvp_node_t
{
"""

header_end = \
"""\
    NWZ_NVP_COUNT /* Number of nvp nodes */
};

/* Invalid NVP index */
#define NWZ_NVP_INVALID     -1 /* Non-existent entry */
/* Number of models */
#define NWZ_MODEL_COUNT     %s
/* Number of series */
#define NWZ_SERIES_COUNT    %s

/* NVP node info */
struct nwz_nvp_info_t
{
    const char *name; /* Sony's name: "bti" */
    unsigned long size; /* Size in bytes */
    const char *desc; /* Description: "bootloader image" */
};

/* NVP index map (nwz_nvp_node_t -> index) */
typedef int nwz_nvp_index_t[NWZ_NVP_COUNT];

/* Model info */
struct nwz_model_info_t
{
    unsigned long mid; /* Model ID: first 4 bytes of the NVP mid entry */
    const char *name; /* Human name: "NWZ-E463" */
};

/* Series info */
struct nwz_series_info_t
{
    const char *codename; /* Rockbox codename: nwz-e460 */
    const char *name; /* Human name: "NWZ-E460 Series" */
    int mid_count; /* number of entries in mid_list */
    unsigned long *mid; /* List of model IDs */
    /* Pointer to a name -> index map, nonexistent entries map to NWZ_NVP_INVALID */
    nwz_nvp_index_t *nvp_index;
};

/* List of all NVP entries, indexed by nwz_nvp_node_t */
extern struct nwz_nvp_info_t nwz_nvp[NWZ_NVP_COUNT];
/* List of all models, sorted by increasing values of model ID */
extern struct nwz_model_info_t nwz_model[NWZ_MODEL_COUNT];
/* List of all series */
extern struct nwz_series_info_t nwz_series[NWZ_SERIES_COUNT];

#endif /* __NWZ_DB_H__ */
"""

with open("nwz_db.h", "w") as fp:
    fp.write(header_begin)
    # generate list of all nvp nodes
    for name in sorted(g_nvp_names):
        # create comment to explain the meaning, gather several meaning together
        # if there are more than one (sorted to keep a stable order when we update)
        explain = ""
        if name in g_nvp_desc:
            explain = " | ".join(sorted(list(g_nvp_desc[name])))
        # overwrite desc set with a single string for later
        g_nvp_desc[name] = explain
        fp.write("    NWZ_NVP_%s, /* %s */\n" % (name.upper(), explain))
    fp.write(header_end % (len(g_models), len(g_series)))

#
# generate tables
#
impl_begin = \
"""\
/***************************************************************************
 *             __________               __   ___.
 *   Open      \______   \ ____   ____ |  | _\_ |__   _______  ___
 *   Source     |       _//  _ \_/ ___\|  |/ /| __ \ /  _ \  \/  /
 *   Jukebox    |    |   (  <_> )  \___|    < | \_\ (  <_> > <  <
 *   Firmware   |____|_  /\____/ \___  >__|_ \|___  /\____/__/\_ \\
 *                     \/            \/     \/    \/            \/
 *
 * Copyright (C) 2016 Amaury Pouly
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version 2
 * of the License, or (at your option) any later version.
 *
 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
 * KIND, either express or implied.
 *
 ****************************************************************************/

/** /!\ This file was automatically generated, DO NOT MODIFY IT DIRECTLY /!\ */

#include "nwz_db.h"

struct nwz_model_info_t nwz_model[NWZ_MODEL_COUNT] =
{
"""

def by_mid(model):
    return model["mid"]

def by_name(nvp_entry):
    return nvp_entry["name"]

def codename_to_c(codename):
    return re.sub('[^a-zA-Z0-9]', '_', codename, 0)

with open("nwz_db.c", "w") as fp:
    fp.write(impl_begin)
    # generate model list (sort by mid)
    for model in sorted(g_models, key = by_mid):
        fp.write("    { %s, \"%s\" },\n" % (hex(model["mid"]), model["name"]))
    fp.write("};\n")
    # generate nvps
    for hash in sorted(g_hash_nvp):
        nvp = g_hash_nvp[hash]
        fp.write("\nstatic int nvp_index_%s[NWZ_NVP_COUNT] =\n" % hash)
        fp.write("{\n")
        for name in sorted(g_nvp_names):
            index = "NWZ_NVP_INVALID"
            if name in nvp:
                index = nvp[name]
            fp.write("    [NWZ_NVP_%s] = %s,\n" % (name.upper(), index))
        fp.write("};\n")
    # generate nvp info
    fp.write("\nstruct nwz_nvp_info_t nwz_nvp[NWZ_NVP_COUNT] =\n")
    fp.write("{\n")
    for name in sorted(g_nvp_names):
        size = 0
        if name in g_nvp_size:
            size_set = g_nvp_size[name]
            if len(size_set) == 0:
                size = 0
            elif len(size_set) == 1:
                size = next(iter(size_set))
            else:
                print("Warning: nvp node \"%s\" has several possible sizes: %s"
                    % (name, size_set))
                size = 0
        desc = ""
        if name in g_nvp_desc:
            desc = g_nvp_desc[name]
        fp.write("    [NWZ_NVP_%s] = { \"%s\", %s, \"%s\" },\n" % (name.upper(),
            name, size, desc))
    fp.write("};\n")
    # generate list of models for each series
    for series in g_series:
        c_codename = codename_to_c(series["codename"])
        list = [hex(mid) for mid in series["models"]]
        limit = 3
        c_list = ""
        while len(list) != 0:
            if len(list) <= limit:
                c_list = c_list + ", ".join(list)
                list = []
            else:
                c_list = c_list + ", ".join(list[:limit]) + ",\n    "
                list = list[limit:]
                limit = 6
        fp.write("\nstatic unsigned long models_%s[] = { %s };\n" % (c_codename, c_list))
    # generate series list
    fp.write("\nstruct nwz_series_info_t nwz_series[NWZ_SERIES_COUNT] =\n{\n")
    for series in g_series:
        name = series["name"]
        codename = series["codename"]
        c_codename = codename_to_c(codename)
        nvp = "0"
        if codename in g_nvp_hash:
            nvp = "&nvp_index_%s" % g_nvp_hash[codename]
        fp.write("    { \"%s\", \"%s\", %s, models_%s, %s },\n" % (codename,
            name, len(series["models"]), c_codename, nvp))
    fp.write("};\n")