3 Copyright (c) 2011-2013 ARM Limited
5 Licensed under the Apache License, Version 2.0 (the "License");
6 you may not use this file except in compliance with the License.
7 You may obtain a copy of the License at
9 http://www.apache.org/licenses/LICENSE-2.0
11 Unless required by applicable law or agreed to in writing, software
12 distributed under the License is distributed on an "AS IS" BASIS,
13 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 See the License for the specific language governing permissions and
15 limitations under the License.
20 from os import stat, walk
22 from time import time, sleep
23 from types import ListType
24 from shutil import copyfile
25 from os.path import join, splitext, exists, relpath, dirname, basename, split
26 from inspect import getmro
28 from multiprocessing import Pool, cpu_count
29 from workspace_tools.utils import run_cmd, mkdir, rel_path, ToolException, split_path
30 from workspace_tools.settings import BUILD_OPTIONS, MBED_ORG_USER
31 import workspace_tools.hooks as hooks
34 #Disables multiprocessing if set to higher number than the host machine CPUs
37 def print_notify(event, silent=False):
38 """ Default command line notification
40 if event['type'] in ['info', 'debug']:
41 print event['message']
43 elif event['type'] == 'cc':
44 event['severity'] = event['severity'].title()
45 event['file'] = basename(event['file'])
46 print '[%(severity)s] %(file)s@%(line)s: %(message)s' % event
48 elif event['type'] == 'progress':
50 print '%s: %s' % (event['action'].title(), basename(event['file']))
52 def print_notify_verbose(event, silent=False):
53 """ Default command line notification with more verbose mode
55 if event['type'] in ['info', 'debug']:
56 print_notify(event) # standard handle
58 elif event['type'] == 'cc':
59 event['severity'] = event['severity'].title()
60 event['file'] = basename(event['file'])
61 event['mcu_name'] = "None"
62 event['toolchain'] = "None"
63 event['target_name'] = event['target_name'].upper() if event['target_name'] else "Unknown"
64 event['toolchain_name'] = event['toolchain_name'].upper() if event['toolchain_name'] else "Unknown"
65 print '[%(severity)s] %(target_name)s::%(toolchain_name)s::%(file)s@%(line)s: %(message)s' % event
67 elif event['type'] == 'progress':
68 print_notify(event) # standard handle
70 def compile_worker(job):
72 for command in job['commands']:
73 _, _stderr, _rc = run_cmd(command, job['work_dir'])
81 'source': job['source'],
82 'object': job['object'],
83 'commands': job['commands'],
88 def __init__(self, base_path=None):
89 self.base_path = base_path
98 self.lib_dirs = set([])
109 self.linker_script = None
115 def add(self, resources):
116 self.inc_dirs += resources.inc_dirs
117 self.headers += resources.headers
119 self.s_sources += resources.s_sources
120 self.c_sources += resources.c_sources
121 self.cpp_sources += resources.cpp_sources
123 self.lib_dirs |= resources.lib_dirs
124 self.objects += resources.objects
125 self.libraries += resources.libraries
127 self.lib_builds += resources.lib_builds
128 self.lib_refs += resources.lib_refs
130 self.repo_dirs += resources.repo_dirs
131 self.repo_files += resources.repo_files
133 if resources.linker_script is not None:
134 self.linker_script = resources.linker_script
136 self.hex_files += resources.hex_files
137 self.bin_files += resources.bin_files
139 def relative_to(self, base, dot=False):
140 for field in ['inc_dirs', 'headers', 's_sources', 'c_sources',
141 'cpp_sources', 'lib_dirs', 'objects', 'libraries',
142 'lib_builds', 'lib_refs', 'repo_dirs', 'repo_files', 'hex_files', 'bin_files']:
143 v = [rel_path(f, base, dot) for f in getattr(self, field)]
144 setattr(self, field, v)
145 if self.linker_script is not None:
146 self.linker_script = rel_path(self.linker_script, base, dot)
148 def win_to_unix(self):
149 for field in ['inc_dirs', 'headers', 's_sources', 'c_sources',
150 'cpp_sources', 'lib_dirs', 'objects', 'libraries',
151 'lib_builds', 'lib_refs', 'repo_dirs', 'repo_files', 'hex_files', 'bin_files']:
152 v = [f.replace('\\', '/') for f in getattr(self, field)]
153 setattr(self, field, v)
154 if self.linker_script is not None:
155 self.linker_script = self.linker_script.replace('\\', '/')
160 for (label, resources) in (
161 ('Include Directories', self.inc_dirs),
162 ('Headers', self.headers),
164 ('Assembly sources', self.s_sources),
165 ('C sources', self.c_sources),
166 ('C++ sources', self.cpp_sources),
168 ('Library directories', self.lib_dirs),
169 ('Objects', self.objects),
170 ('Libraries', self.libraries),
172 ('Hex files', self.hex_files),
173 ('Bin files', self.bin_files),
176 s.append('%s:\n ' % label + '\n '.join(resources))
178 if self.linker_script:
179 s.append('Linker Script: ' + self.linker_script)
184 # Support legacy build conventions: the original mbed build system did not have
185 # standard labels for the "TARGET_" and "TOOLCHAIN_" specific directories, but
186 # had the knowledge of a list of these directories to be ignored.
187 LEGACY_IGNORE_DIRS = set([
188 'LPC11U24', 'LPC1768', 'LPC2368', 'LPC4088', 'LPC812', 'KL25Z',
189 'ARM', 'GCC_ARM', 'GCC_CR', 'GCC_CS', 'IAR', 'uARM'
191 LEGACY_TOOLCHAIN_NAMES = {
192 'ARM_STD':'ARM', 'ARM_MICRO': 'uARM',
193 'GCC_ARM': 'GCC_ARM', 'GCC_CR': 'GCC_CR', 'GCC_CS': 'GCC_CS',
202 "Cortex-M0" : ["__CORTEX_M0", "ARM_MATH_CM0"],
203 "Cortex-M0+": ["__CORTEX_M0PLUS", "ARM_MATH_CM0PLUS"],
204 "Cortex-M1" : ["__CORTEX_M3", "ARM_MATH_CM1"],
205 "Cortex-M3" : ["__CORTEX_M3", "ARM_MATH_CM3"],
206 "Cortex-M4" : ["__CORTEX_M4", "ARM_MATH_CM4"],
207 "Cortex-M4F" : ["__CORTEX_M4", "ARM_MATH_CM4", "__FPU_PRESENT=1"],
208 "Cortex-M7" : ["__CORTEX_M7", "ARM_MATH_CM7"],
209 "Cortex-M7F" : ["__CORTEX_M7", "ARM_MATH_CM7", "__FPU_PRESENT=1"],
210 "Cortex-A9" : ["__CORTEX_A9", "ARM_MATH_CA9", "__FPU_PRESENT", "__CMSIS_RTOS", "__EVAL", "__MBED_CMSIS_RTOS_CA9"],
213 GOANNA_FORMAT = "[Goanna] warning [%FILENAME%:%LINENO%] - [%CHECKNAME%(%SEVERITY%)] %MESSAGE%"
214 GOANNA_DIAGNOSTIC_PATTERN = re.compile(r'"\[Goanna\] (?P<severity>warning) \[(?P<file>[^:]+):(?P<line>\d+)\] \- (?P<message>.*)"')
216 def __init__(self, target, options=None, notify=None, macros=None, silent=False):
218 self.name = self.__class__.__name__
219 self.hook = hooks.Hook(target, self)
222 self.legacy_ignore_dirs = LEGACY_IGNORE_DIRS - set([target.name, LEGACY_TOOLCHAIN_NAMES[self.name]])
224 self.notify_fun = notify if notify is not None else print_notify
225 self.options = options if options is not None else []
227 self.macros = macros or []
228 self.options.extend(BUILD_OPTIONS)
230 self.info("Build Options: %s" % (', '.join(self.options)))
232 self.obj_path = join("TARGET_"+target.name, "TOOLCHAIN_"+self.name)
236 self.has_config = False
238 self.build_all = False
239 self.timestamp = time()
246 def notify(self, event):
247 """ Little closure for notify functions
249 return self.notify_fun(event, self.silent)
252 if self.mp_pool is not None:
253 self.mp_pool.terminate()
255 def goanna_parse_line(self, line):
256 if "analyze" in self.options:
257 return self.GOANNA_DIAGNOSTIC_PATTERN.match(line)
261 def get_symbols(self):
262 if self.symbols is None:
263 # Target and Toolchain symbols
264 labels = self.get_labels()
265 self.symbols = ["TARGET_%s" % t for t in labels['TARGET']]
266 self.symbols.extend(["TOOLCHAIN_%s" % t for t in labels['TOOLCHAIN']])
270 self.symbols.append('HAVE_MBED_CONFIG_H')
273 if self.target.core in mbedToolchain.CORTEX_SYMBOLS:
274 self.symbols.extend(mbedToolchain.CORTEX_SYMBOLS[self.target.core])
276 # Symbols defined by the on-line build.system
277 self.symbols.extend(['MBED_BUILD_TIMESTAMP=%s' % self.timestamp, '__MBED__=1'])
279 self.symbols.append('MBED_USERNAME=' + MBED_ORG_USER)
281 # Add target's symbols
282 self.symbols += self.target.macros
283 # Add extra symbols passed via 'macros' parameter
284 self.symbols += self.macros
286 # Form factor variables
287 if hasattr(self.target, 'supported_form_factors'):
288 self.symbols.extend(["TARGET_FF_%s" % t for t in self.target.supported_form_factors])
292 def get_labels(self):
293 if self.labels is None:
294 toolchain_labels = [c.__name__ for c in getmro(self.__class__)]
295 toolchain_labels.remove('mbedToolchain')
297 'TARGET': self.target.get_labels(),
298 'TOOLCHAIN': toolchain_labels
302 def need_update(self, target, dependencies):
306 if not exists(target):
309 target_mod_time = stat(target).st_mtime
311 for d in dependencies:
313 # Some objects are not provided with full path and here we do not have
314 # information about the library paths. Safe option: assume an update
315 if not d or not exists(d):
318 if stat(d).st_mtime >= target_mod_time:
323 def scan_resources(self, path):
324 labels = self.get_labels()
325 resources = Resources(path)
326 self.has_config = False
328 """ os.walk(top[, topdown=True[, onerror=None[, followlinks=False]]])
329 When topdown is True, the caller can modify the dirnames list in-place
330 (perhaps using del or slice assignment), and walk() will only recurse into
331 the subdirectories whose names remain in dirnames; this can be used to prune
332 the search, impose a specific order of visiting, or even to inform walk()
333 about directories the caller creates or renames before it resumes walk()
334 again. Modifying dirnames when topdown is False is ineffective, because in
335 bottom-up mode the directories in dirnames are generated before dirpath
338 for root, dirs, files in walk(path):
339 # Remove ignored directories
342 dir_path = join(root, d)
343 resources.repo_dirs.append(dir_path)
344 resources.repo_files.extend(self.scan_repository(dir_path))
346 if ((d.startswith('.') or d in self.legacy_ignore_dirs) or
347 (d.startswith('TARGET_') and d[7:] not in labels['TARGET']) or
348 (d.startswith('TOOLCHAIN_') and d[10:] not in labels['TOOLCHAIN'])):
351 # Add root to include paths
352 resources.inc_dirs.append(root)
355 file_path = join(root, file)
356 _, ext = splitext(file)
360 resources.s_sources.append(file_path)
363 resources.c_sources.append(file_path)
366 resources.cpp_sources.append(file_path)
368 elif ext == '.h' or ext == '.hpp':
369 if basename(file_path) == "mbed_config.h":
370 self.has_config = True
371 resources.headers.append(file_path)
374 resources.objects.append(file_path)
376 elif ext == self.LIBRARY_EXT:
377 resources.libraries.append(file_path)
378 resources.lib_dirs.add(root)
380 elif ext == self.LINKER_EXT:
381 if resources.linker_script is not None:
382 self.info("Warning: Multiple linker scripts detected: %s -> %s" % (resources.linker_script, file_path))
383 resources.linker_script = file_path
386 resources.lib_refs.append(file_path)
389 resources.lib_builds.append(file_path)
391 elif file == '.hgignore':
392 resources.repo_files.append(file_path)
395 resources.hex_files.append(file_path)
398 resources.bin_files.append(file_path)
402 def scan_repository(self, path):
405 for root, dirs, files in walk(path):
406 # Remove ignored directories
408 if d == '.' or d == '..':
412 file_path = join(root, file)
413 resources.append(file_path)
417 def copy_files(self, files_paths, trg_path, rel_path=None):
418 # Handle a single file
419 if type(files_paths) != ListType: files_paths = [files_paths]
421 for source in files_paths:
423 files_paths.remove(source)
425 for source in files_paths:
426 if rel_path is not None:
427 relative_path = relpath(source, rel_path)
429 _, relative_path = split(source)
431 target = join(trg_path, relative_path)
433 if (target != source) and (self.need_update(target, [source])):
434 self.progress("copy", relative_path)
435 mkdir(dirname(target))
436 copyfile(source, target)
438 def relative_object_path(self, build_path, base_dir, source):
439 source_dir, name, _ = split_path(source)
440 obj_dir = join(build_path, relpath(source_dir, base_dir))
442 return join(obj_dir, name + '.o')
444 def compile_sources(self, resources, build_path, inc_dirs=None):
445 # Web IDE progress bar for project build
446 files_to_compile = resources.s_sources + resources.c_sources + resources.cpp_sources
447 self.to_be_compiled = len(files_to_compile)
450 #for i in self.build_params:
452 # self.debug("%s" % self.build_params[i])
454 inc_paths = resources.inc_dirs
455 if inc_dirs is not None:
456 inc_paths.extend(inc_dirs)
462 # The dependency checking for C/C++ is delegated to the compiler
463 base_path = resources.base_path
464 files_to_compile.sort()
465 for source in files_to_compile:
466 _, name, _ = split_path(source)
467 object = self.relative_object_path(build_path, base_path, source)
469 # Avoid multiple mkdir() calls on same work directory
470 work_dir = dirname(object)
471 if work_dir is not prev_dir:
475 # Queue mode (multiprocessing)
476 commands = self.compile_command(source, object, inc_paths)
477 if commands is not None:
481 'commands': commands,
482 'work_dir': work_dir,
483 'chroot': self.CHROOT
486 objects.append(object)
488 # Use queues/multiprocessing if cpu count is higher than setting
489 jobs = self.jobs if self.jobs else cpu_count()
490 if jobs > CPU_COUNT_MIN and len(queue) > jobs:
491 return self.compile_queue(queue, objects)
493 return self.compile_seq(queue, objects)
495 def compile_seq(self, queue, objects):
497 result = compile_worker(item)
500 self.progress("compile", item['source'], build_update=True)
501 for res in result['results']:
502 self.debug("Command: %s" % ' '.join(res['command']))
503 self.compile_output([
508 objects.append(result['object'])
511 def compile_queue(self, queue, objects):
512 jobs_count = int(self.jobs if self.jobs else cpu_count())
513 p = Pool(processes=jobs_count)
516 for i in range(len(queue)):
517 results.append(p.apply_async(compile_worker, [queue[i]]))
525 raise ToolException("Compile did not finish in 5 minutes")
535 self.progress("compile", result['source'], build_update=True)
536 for res in result['results']:
537 self.debug("Command: %s" % ' '.join(res['command']))
538 self.compile_output([
543 objects.append(result['object'])
544 except ToolException, err:
547 raise ToolException(err)
550 if pending > jobs_count:
554 if len(results) == 0:
565 def compile_command(self, source, object, includes):
567 _, ext = splitext(source)
570 if ext == '.c' or ext == '.cpp':
571 base, _ = splitext(object)
572 dep_path = base + '.d'
573 deps = self.parse_dependencies(dep_path) if (exists(dep_path)) else []
574 if len(deps) == 0 or self.need_update(object, deps):
576 return self.compile_c(source, object, includes)
578 return self.compile_cpp(source, object, includes)
581 if self.need_update(object, deps):
582 return self.assemble(source, object, includes)
588 def compile_output(self, output=[]):
593 # Parse output for Warnings and Errors
594 self.parse_output(_stderr)
595 self.debug("Return: %s"% _rc)
596 for error_line in _stderr.splitlines():
597 self.debug("Output: %s"% error_line)
601 for line in _stderr.splitlines():
602 self.tool_error(line)
603 raise ToolException(_stderr)
605 def compile(self, cc, source, object, includes):
606 _, ext = splitext(source)
609 command = cc + ['-D%s' % s for s in self.get_symbols()] + ["-I%s" % i for i in includes] + ["-o", object, source]
611 if hasattr(self, "get_dep_opt"):
612 base, _ = splitext(object)
613 dep_path = base + '.d'
614 command.extend(self.get_dep_opt(dep_path))
616 if hasattr(self, "cc_extra"):
617 command.extend(self.cc_extra(base))
621 def compile_c(self, source, object, includes):
622 return self.compile(self.cc, source, object, includes)
624 def compile_cpp(self, source, object, includes):
625 return self.compile(self.cppc, source, object, includes)
627 def build_library(self, objects, dir, name):
628 lib = self.STD_LIB_NAME % name
629 fout = join(dir, lib)
630 if self.need_update(fout, objects):
631 self.info("Library: %s" % lib)
632 self.archive(objects, fout)
634 def link_program(self, r, tmp_path, name):
636 if hasattr(self.target, 'OUTPUT_EXT'):
637 ext = self.target.OUTPUT_EXT
639 if hasattr(self.target, 'OUTPUT_NAMING'):
640 self.var("binary_naming", self.target.OUTPUT_NAMING)
641 if self.target.OUTPUT_NAMING == "8.3":
645 filename = name+'.'+ext
646 elf = join(tmp_path, name + '.elf')
647 bin = join(tmp_path, filename)
649 if self.need_update(elf, r.objects + r.libraries + [r.linker_script]):
650 self.progress("link", name)
651 self.link(elf, r.objects, r.libraries, r.lib_dirs, r.linker_script)
653 if self.need_update(bin, [elf]):
654 self.progress("elf2bin", name)
656 self.binary(r, elf, bin)
658 self.var("compile_succeded", True)
659 self.var("binary", filename)
663 def default_cmd(self, command):
664 _stdout, _stderr, _rc = run_cmd(command)
665 # Print all warning / erros from stderr to console output
666 for error_line in _stderr.splitlines():
669 self.debug("Command: %s"% ' '.join(command))
670 self.debug("Return: %s"% _rc)
672 for output_line in _stdout.splitlines():
673 self.debug("Output: %s"% output_line)
674 for error_line in _stderr.splitlines():
675 self.debug("Errors: %s"% error_line)
678 for line in _stderr.splitlines():
679 self.tool_error(line)
680 raise ToolException(_stderr)
682 ### NOTIFICATIONS ###
683 def info(self, message):
684 self.notify({'type': 'info', 'message': message})
686 def debug(self, message):
688 if type(message) is ListType:
689 message = ' '.join(message)
690 message = "[DEBUG] " + message
691 self.notify({'type': 'debug', 'message': message})
693 def cc_info(self, severity, file, line, message, target_name=None, toolchain_name=None):
694 self.notify({'type': 'cc',
695 'severity': severity,
699 'target_name': target_name,
700 'toolchain_name': toolchain_name})
702 def progress(self, action, file, build_update=False):
703 msg = {'type': 'progress', 'action': action, 'file': file}
705 msg['percent'] = 100. * float(self.compiled) / float(self.to_be_compiled)
708 def tool_error(self, message):
709 self.notify({'type': 'tool_error', 'message': message})
711 def var(self, key, value):
712 self.notify({'type': 'var', 'key': key, 'val': value})
714 from workspace_tools.settings import ARM_BIN
715 from workspace_tools.settings import GCC_ARM_PATH, GCC_CR_PATH, GCC_CS_PATH, CW_EWL_PATH, CW_GCC_PATH
716 from workspace_tools.settings import IAR_PATH
718 TOOLCHAIN_BIN_PATH = {
721 'GCC_ARM': GCC_ARM_PATH,
722 'GCC_CS': GCC_CS_PATH,
723 'GCC_CR': GCC_CR_PATH,
724 'GCC_CW_EWL': CW_EWL_PATH,
725 'GCC_CW_NEWLIB': CW_GCC_PATH,
729 from workspace_tools.toolchains.arm import ARM_STD, ARM_MICRO
730 from workspace_tools.toolchains.gcc import GCC_ARM, GCC_CS, GCC_CR
731 from workspace_tools.toolchains.gcc import GCC_CW_EWL, GCC_CW_NEWLIB
732 from workspace_tools.toolchains.iar import IAR
734 TOOLCHAIN_CLASSES = {
740 'GCC_CW_EWL': GCC_CW_EWL,
741 'GCC_CW_NEWLIB': GCC_CW_NEWLIB,
745 TOOLCHAINS = set(TOOLCHAIN_CLASSES.keys())