1 #!/usr/bin/env python3
2 # SPDX-License-Identifier: GPL-2.0+
3 #
4 # Author: Masahiro Yamada <yamada.m@jp.panasonic.com>
5 #
7 """
8 Converter from Kconfig and MAINTAINERS to a board database.
10 Run 'tools/genboardscfg.py' to create a board database.
12 Run 'tools/genboardscfg.py -h' for available options.
13 """
15 import errno
16 import fnmatch
17 import glob
18 import multiprocessing
19 import optparse
20 import os
21 import sys
22 import tempfile
23 import time
25 from buildman import kconfiglib
27 ### constant variables ###
28 OUTPUT_FILE = 'boards.cfg'
29 CONFIG_DIR = 'configs'
30 SLEEP_TIME = 0.03
31 COMMENT_BLOCK = '''#
32 # List of boards
33 # Automatically generated by %s: don't edit
34 #
35 # Status, Arch, CPU, SoC, Vendor, Board, Target, Options, Maintainers
37 ''' % __file__
39 ### helper functions ###
40 def try_remove(f):
41 """Remove a file ignoring 'No such file or directory' error."""
42 try:
43 os.remove(f)
44 except OSError as exception:
45 # Ignore 'No such file or directory' error
46 if exception.errno != errno.ENOENT:
47 raise
49 def check_top_directory():
50 """Exit if we are not at the top of source directory."""
51 for f in ('README', 'Licenses'):
52 if not os.path.exists(f):
53 sys.exit('Please run at the top of source directory.')
55 def output_is_new(output):
56 """Check if the output file is up to date.
58 Returns:
59 True if the given output file exists and is newer than any of
60 *_defconfig, MAINTAINERS and Kconfig*. False otherwise.
61 """
62 try:
63 ctime = os.path.getctime(output)
64 except OSError as exception:
65 if exception.errno == errno.ENOENT:
66 # return False on 'No such file or directory' error
67 return False
68 else:
69 raise
71 for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR):
72 for filename in fnmatch.filter(filenames, '*_defconfig'):
73 if fnmatch.fnmatch(filename, '.*'):
74 continue
75 filepath = os.path.join(dirpath, filename)
76 if ctime < os.path.getctime(filepath):
77 return False
79 for (dirpath, dirnames, filenames) in os.walk('.'):
80 for filename in filenames:
81 if (fnmatch.fnmatch(filename, '*~') or
82 not fnmatch.fnmatch(filename, 'Kconfig*') and
83 not filename == 'MAINTAINERS'):
84 continue
85 filepath = os.path.join(dirpath, filename)
86 if ctime < os.path.getctime(filepath):
87 return False
89 # Detect a board that has been removed since the current board database
90 # was generated
91 with open(output, encoding="utf-8") as f:
92 for line in f:
93 if line[0] == '#' or line == '\n':
94 continue
95 defconfig = line.split()[6] + '_defconfig'
96 if not os.path.exists(os.path.join(CONFIG_DIR, defconfig)):
97 return False
99 return True
101 ### classes ###
102 class KconfigScanner:
104 """Kconfig scanner."""
106 ### constant variable only used in this class ###
107 _SYMBOL_TABLE = {
108 'arch' : 'SYS_ARCH',
109 'cpu' : 'SYS_CPU',
110 'soc' : 'SYS_SOC',
111 'vendor' : 'SYS_VENDOR',
112 'board' : 'SYS_BOARD',
113 'config' : 'SYS_CONFIG_NAME',
114 'options' : 'SYS_EXTRA_OPTIONS'
115 }
117 def __init__(self):
118 """Scan all the Kconfig files and create a Kconfig object."""
119 # Define environment variables referenced from Kconfig
120 os.environ['srctree'] = os.getcwd()
121 os.environ['UBOOTVERSION'] = 'dummy'
122 os.environ['KCONFIG_OBJDIR'] = ''
123 self._conf = kconfiglib.Kconfig(warn=False)
125 def __del__(self):
126 """Delete a leftover temporary file before exit.
128 The scan() method of this class creates a temporay file and deletes
129 it on success. If scan() method throws an exception on the way,
130 the temporary file might be left over. In that case, it should be
131 deleted in this destructor.
132 """
133 if hasattr(self, '_tmpfile') and self._tmpfile:
134 try_remove(self._tmpfile)
136 def scan(self, defconfig):
137 """Load a defconfig file to obtain board parameters.
139 Arguments:
140 defconfig: path to the defconfig file to be processed
142 Returns:
143 A dictionary of board parameters. It has a form of:
144 {
145 'arch': <arch_name>,
146 'cpu': <cpu_name>,
147 'soc': <soc_name>,
148 'vendor': <vendor_name>,
149 'board': <board_name>,
150 'target': <target_name>,
151 'config': <config_header_name>,
152 'options': <extra_options>
153 }
154 """
155 # strip special prefixes and save it in a temporary file
156 fd, self._tmpfile = tempfile.mkstemp()
157 with os.fdopen(fd, 'w') as f:
158 for line in open(defconfig):
159 colon = line.find(':CONFIG_')
160 if colon == -1:
161 f.write(line)
162 else:
163 f.write(line[colon + 1:])
165 self._conf.load_config(self._tmpfile)
166 try_remove(self._tmpfile)
167 self._tmpfile = None
169 params = {}
171 # Get the value of CONFIG_SYS_ARCH, CONFIG_SYS_CPU, ... etc.
172 # Set '-' if the value is empty.
173 for key, symbol in list(self._SYMBOL_TABLE.items()):
174 value = self._conf.syms.get(symbol).str_value
175 if value:
176 params[key] = value
177 else:
178 params[key] = '-'
180 defconfig = os.path.basename(defconfig)
181 params['target'], match, rear = defconfig.partition('_defconfig')
182 assert match and not rear, '%s : invalid defconfig' % defconfig
184 # fix-up for aarch64
185 if params['arch'] == 'arm' and params['cpu'] == 'armv8':
186 params['arch'] = 'aarch64'
188 # fix-up options field. It should have the form:
189 # <config name>[:comma separated config options]
190 if params['options'] != '-':
191 params['options'] = params['config'] + ':' + \
192 params['options'].replace(r'\"', '"')
193 elif params['config'] != params['target']:
194 params['options'] = params['config']
196 return params
198 def scan_defconfigs_for_multiprocess(queue, defconfigs):
199 """Scan defconfig files and queue their board parameters
201 This function is intended to be passed to
202 multiprocessing.Process() constructor.
204 Arguments:
205 queue: An instance of multiprocessing.Queue().
206 The resulting board parameters are written into it.
207 defconfigs: A sequence of defconfig files to be scanned.
208 """
209 kconf_scanner = KconfigScanner()
210 for defconfig in defconfigs:
211 queue.put(kconf_scanner.scan(defconfig))
213 def read_queues(queues, params_list):
214 """Read the queues and append the data to the paramers list"""
215 for q in queues:
216 while not q.empty():
217 params_list.append(q.get())
219 def scan_defconfigs(jobs=1):
220 """Collect board parameters for all defconfig files.
222 This function invokes multiple processes for faster processing.
224 Arguments:
225 jobs: The number of jobs to run simultaneously
226 """
227 all_defconfigs = []
228 for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR):
229 for filename in fnmatch.filter(filenames, '*_defconfig'):
230 if fnmatch.fnmatch(filename, '.*'):
231 continue
232 all_defconfigs.append(os.path.join(dirpath, filename))
234 total_boards = len(all_defconfigs)
235 processes = []
236 queues = []
237 for i in range(jobs):
238 defconfigs = all_defconfigs[total_boards * i // jobs :
239 total_boards * (i + 1) // jobs]
240 q = multiprocessing.Queue(maxsize=-1)
241 p = multiprocessing.Process(target=scan_defconfigs_for_multiprocess,
242 args=(q, defconfigs))
243 p.start()
244 processes.append(p)
245 queues.append(q)
247 # The resulting data should be accumulated to this list
248 params_list = []
250 # Data in the queues should be retrieved preriodically.
251 # Otherwise, the queues would become full and subprocesses would get stuck.
252 while any([p.is_alive() for p in processes]):
253 read_queues(queues, params_list)
254 # sleep for a while until the queues are filled
255 time.sleep(SLEEP_TIME)
257 # Joining subprocesses just in case
258 # (All subprocesses should already have been finished)
259 for p in processes:
260 p.join()
262 # retrieve leftover data
263 read_queues(queues, params_list)
265 return params_list
267 class MaintainersDatabase:
269 """The database of board status and maintainers."""
271 def __init__(self):
272 """Create an empty database."""
273 self.database = {}
275 def get_status(self, target):
276 """Return the status of the given board.
278 The board status is generally either 'Active' or 'Orphan'.
279 Display a warning message and return '-' if status information
280 is not found.
282 Returns:
283 'Active', 'Orphan' or '-'.
284 """
285 if not target in self.database:
286 print("WARNING: no status info for '%s'" % target, file=sys.stderr)
287 return '-'
289 tmp = self.database[target][0]
290 if tmp.startswith('Maintained'):
291 return 'Active'
292 elif tmp.startswith('Supported'):
293 return 'Active'
294 elif tmp.startswith('Orphan'):
295 return 'Orphan'
296 else:
297 print(("WARNING: %s: unknown status for '%s'" %
298 (tmp, target)), file=sys.stderr)
299 return '-'
301 def get_maintainers(self, target):
302 """Return the maintainers of the given board.
304 Returns:
305 Maintainers of the board. If the board has two or more maintainers,
306 they are separated with colons.
307 """
308 if not target in self.database:
309 print("WARNING: no maintainers for '%s'" % target, file=sys.stderr)
310 return ''
312 return ':'.join(self.database[target][1])
314 def parse_file(self, file):
315 """Parse a MAINTAINERS file.
317 Parse a MAINTAINERS file and accumulates board status and
318 maintainers information.
320 Arguments:
321 file: MAINTAINERS file to be parsed
322 """
323 targets = []
324 maintainers = []
325 status = '-'
326 for line in open(file, encoding="utf-8"):
327 # Check also commented maintainers
328 if line[:3] == '#M:':
329 line = line[1:]
330 tag, rest = line[:2], line[2:].strip()
331 if tag == 'M:':
332 maintainers.append(rest)
333 elif tag == 'F:':
334 # expand wildcard and filter by 'configs/*_defconfig'
335 for f in glob.glob(rest):
336 front, match, rear = f.partition('configs/')
337 if not front and match:
338 front, match, rear = rear.rpartition('_defconfig')
339 if match and not rear:
340 targets.append(front)
341 elif tag == 'S:':
342 status = rest
343 elif line == '\n':
344 for target in targets:
345 self.database[target] = (status, maintainers)
346 targets = []
347 maintainers = []
348 status = '-'
349 if targets:
350 for target in targets:
351 self.database[target] = (status, maintainers)
353 def insert_maintainers_info(params_list):
354 """Add Status and Maintainers information to the board parameters list.
356 Arguments:
357 params_list: A list of the board parameters
358 """
359 database = MaintainersDatabase()
360 for (dirpath, dirnames, filenames) in os.walk('.'):
361 if 'MAINTAINERS' in filenames:
362 database.parse_file(os.path.join(dirpath, 'MAINTAINERS'))
364 for i, params in enumerate(params_list):
365 target = params['target']
366 params['status'] = database.get_status(target)
367 params['maintainers'] = database.get_maintainers(target)
368 params_list[i] = params
370 def format_and_output(params_list, output):
371 """Write board parameters into a file.
373 Columnate the board parameters, sort lines alphabetically,
374 and then write them to a file.
376 Arguments:
377 params_list: The list of board parameters
378 output: The path to the output file
379 """
380 FIELDS = ('status', 'arch', 'cpu', 'soc', 'vendor', 'board', 'target',
381 'options', 'maintainers')
383 # First, decide the width of each column
384 max_length = dict([ (f, 0) for f in FIELDS])
385 for params in params_list:
386 for f in FIELDS:
387 max_length[f] = max(max_length[f], len(params[f]))
389 output_lines = []
390 for params in params_list:
391 line = ''
392 for f in FIELDS:
393 # insert two spaces between fields like column -t would
394 line += ' ' + params[f].ljust(max_length[f])
395 output_lines.append(line.strip())
397 # ignore case when sorting
398 output_lines.sort(key=str.lower)
400 with open(output, 'w', encoding="utf-8") as f:
401 f.write(COMMENT_BLOCK + '\n'.join(output_lines) + '\n')
403 def gen_boards_cfg(output, jobs=1, force=False, quiet=False):
404 """Generate a board database file.
406 Arguments:
407 output: The name of the output file
408 jobs: The number of jobs to run simultaneously
409 force: Force to generate the output even if it is new
410 quiet: True to avoid printing a message if nothing needs doing
411 """
412 check_top_directory()
414 if not force and output_is_new(output):
415 if not quiet:
416 print("%s is up to date. Nothing to do." % output)
417 sys.exit(0)
419 params_list = scan_defconfigs(jobs)
420 insert_maintainers_info(params_list)
421 format_and_output(params_list, output)
423 def main():
424 try:
425 cpu_count = multiprocessing.cpu_count()
426 except NotImplementedError:
427 cpu_count = 1
429 parser = optparse.OptionParser()
430 # Add options here
431 parser.add_option('-f', '--force', action="store_true", default=False,
432 help='regenerate the output even if it is new')
433 parser.add_option('-j', '--jobs', type='int', default=cpu_count,
434 help='the number of jobs to run simultaneously')
435 parser.add_option('-o', '--output', default=OUTPUT_FILE,
436 help='output file [default=%s]' % OUTPUT_FILE)
437 parser.add_option('-q', '--quiet', action="store_true", help='run silently')
438 (options, args) = parser.parse_args()
440 gen_boards_cfg(options.output, jobs=options.jobs, force=options.force,
441 quiet=options.quiet)
443 if __name__ == '__main__':
444 main()