3 Copyright (c) 2011-2013 ARM Limited
5 Licensed under the Apache License, Version 2.0 (the "License");
6 you may not use this file except in compliance with the License.
7 You may obtain a copy of the License at
9 http://www.apache.org/licenses/LICENSE-2.0
11 Unless required by applicable law or agreed to in writing, software
12 distributed under the License is distributed on an "AS IS" BASIS,
13 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 See the License for the specific language governing permissions and
15 limitations under the License.
23 from types import ListType
24 from shutil import rmtree
25 from os.path import join, exists, basename
27 from workspace_tools.utils import mkdir, run_cmd, run_cmd_ext
28 from workspace_tools.paths import MBED_TARGETS_PATH, MBED_LIBRARIES, MBED_API, MBED_HAL, MBED_COMMON
29 from workspace_tools.targets import TARGET_NAMES, TARGET_MAP
30 from workspace_tools.libraries import Library
31 from workspace_tools.toolchains import TOOLCHAIN_CLASSES
32 from jinja2 import FileSystemLoader
33 from jinja2.environment import Environment
36 def build_project(src_path, build_path, target, toolchain_name,
37 libraries_paths=None, options=None, linker_script=None,
38 clean=False, notify=None, verbose=False, name=None, macros=None, inc_dirs=None, jobs=1, silent=False):
39 """ This function builds project. Project can be for example one test / UT
42 toolchain = TOOLCHAIN_CLASSES[toolchain_name](target, options, notify, macros, silent)
43 toolchain.VERBOSE = verbose
45 toolchain.build_all = clean
46 src_paths = [src_path] if type(src_path) != ListType else src_path
48 # We need to remove all paths which are repeated to avoid
49 # multiple compilations and linking with the same objects
50 src_paths = [src_paths[0]] + list(set(src_paths[1:]))
52 PROJECT_BASENAME = basename(src_paths[0])
55 # We will use default project name based on project folder name
56 name = PROJECT_BASENAME
57 toolchain.info("Building project %s (%s, %s)" % (PROJECT_BASENAME.upper(), target.name, toolchain_name))
59 # User used custom global project name to have the same name for the
60 toolchain.info("Building project %s to %s (%s, %s)" % (PROJECT_BASENAME.upper(), name, target.name, toolchain_name))
62 # Scan src_path and libraries_paths for resources
63 resources = toolchain.scan_resources(src_paths[0])
64 for path in src_paths[1:]:
65 resources.add(toolchain.scan_resources(path))
66 if libraries_paths is not None:
67 src_paths.extend(libraries_paths)
68 for path in libraries_paths:
69 resources.add(toolchain.scan_resources(path))
71 if linker_script is not None:
72 resources.linker_script = linker_script
76 if exists(build_path):
80 # We need to add if necessary additional include directories
82 if type(inc_dirs) == ListType:
83 resources.inc_dirs.extend(inc_dirs)
85 resources.inc_dirs.append(inc_dirs)
88 for path in src_paths:
89 src = toolchain.scan_resources(path)
90 objects = toolchain.compile_sources(src, build_path, resources.inc_dirs)
91 resources.objects.extend(objects)
94 return toolchain.link_program(resources, build_path, name)
97 def build_library(src_paths, build_path, target, toolchain_name,
98 dependencies_paths=None, options=None, name=None, clean=False,
99 notify=None, verbose=False, macros=None, inc_dirs=None, inc_dirs_ext=None, jobs=1, silent=False):
100 """ src_path: the path of the source directory
101 build_path: the path of the build directory
102 target: ['LPC1768', 'LPC11U24', 'LPC2368']
103 toolchain: ['ARM', 'uARM', 'GCC_ARM', 'GCC_CS', 'GCC_CR']
104 library_paths: List of paths to additional libraries
105 clean: Rebuild everything if True
106 notify: Notify function for logs
107 verbose: Write the actual tools command lines if True
108 inc_dirs: additional include directories which should be included in build
109 inc_dirs_ext: additional include directories which should be copied to library directory
111 if type(src_paths) != ListType:
112 src_paths = [src_paths]
114 for src_path in src_paths:
115 if not exists(src_path):
116 raise Exception("The library source folder does not exist: %s", src_path)
119 toolchain = TOOLCHAIN_CLASSES[toolchain_name](target, options, macros=macros, notify=notify, silent=silent)
120 toolchain.VERBOSE = verbose
121 toolchain.jobs = jobs
122 toolchain.build_all = clean
124 # The first path will give the name to the library
125 name = basename(src_paths[0])
126 toolchain.info("Building library %s (%s, %s)" % (name.upper(), target.name, toolchain_name))
130 for src_path in src_paths:
131 resources.append(toolchain.scan_resources(src_path))
133 # Add extra include directories / files which are required by library
134 # This files usually are not in the same directory as source files so
135 # previous scan will not include them
136 if inc_dirs_ext is not None:
137 for inc_ext in inc_dirs_ext:
138 resources.append(toolchain.scan_resources(inc_ext))
140 # Dependencies Include Paths
141 dependencies_include_dir = []
142 if dependencies_paths is not None:
143 for path in dependencies_paths:
144 lib_resources = toolchain.scan_resources(path)
145 dependencies_include_dir.extend(lib_resources.inc_dirs)
148 dependencies_include_dir.extend(inc_dirs)
150 # Create the desired build directory structure
151 bin_path = join(build_path, toolchain.obj_path)
153 tmp_path = join(build_path, '.temp', toolchain.obj_path)
157 for resource in resources:
158 toolchain.copy_files(resource.headers, build_path, rel_path=resource.base_path)
159 dependencies_include_dir.extend(toolchain.scan_resources(build_path).inc_dirs)
163 for resource in resources:
164 objects.extend(toolchain.compile_sources(resource, tmp_path, dependencies_include_dir))
166 toolchain.build_library(objects, bin_path, name)
169 def build_lib(lib_id, target, toolchain, options=None, verbose=False, clean=False, macros=None, notify=None, jobs=1, silent=False):
170 """ Wrapper for build_library function.
171 Function builds library in proper directory using all dependencies and macros defined by user.
173 lib = Library(lib_id)
174 if lib.is_supported(target, toolchain):
175 # We need to combine macros from parameter list with macros from library definition
176 MACROS = lib.macros if lib.macros else []
178 MACROS.extend(macros)
180 build_library(lib.source_dir, lib.build_dir, target, toolchain, lib.dependencies, options,
186 inc_dirs=lib.inc_dirs,
187 inc_dirs_ext=lib.inc_dirs_ext,
190 print 'Library "%s" is not yet supported on target %s with toolchain %s' % (lib_id, target.name, toolchain)
193 # We do have unique legacy conventions about how we build and package the mbed library
194 def build_mbed_libs(target, toolchain_name, options=None, verbose=False, clean=False, macros=None, notify=None, jobs=1, silent=False):
195 """ Function returns True is library was built and false if building was skipped """
196 # Check toolchain support
197 if toolchain_name not in target.supported_toolchains:
198 supported_toolchains_text = ", ".join(target.supported_toolchains)
199 print '%s target is not yet supported by toolchain %s' % (target.name, toolchain_name)
200 print '%s target supports %s toolchain%s' % (target.name, supported_toolchains_text, 's' if len(target.supported_toolchains) > 1 else '')
204 toolchain = TOOLCHAIN_CLASSES[toolchain_name](target, options, macros=macros, notify=notify, silent=silent)
205 toolchain.VERBOSE = verbose
206 toolchain.jobs = jobs
207 toolchain.build_all = clean
209 # Source and Build Paths
210 BUILD_TARGET = join(MBED_LIBRARIES, "TARGET_" + target.name)
211 BUILD_TOOLCHAIN = join(BUILD_TARGET, "TOOLCHAIN_" + toolchain.name)
212 mkdir(BUILD_TOOLCHAIN)
214 TMP_PATH = join(MBED_LIBRARIES, '.temp', toolchain.obj_path)
218 toolchain.info("Building library %s (%s, %s)"% ('CMSIS', target.name, toolchain_name))
219 cmsis_src = join(MBED_TARGETS_PATH, "cmsis")
220 resources = toolchain.scan_resources(cmsis_src)
222 toolchain.copy_files(resources.headers, BUILD_TARGET)
223 toolchain.copy_files(resources.linker_script, BUILD_TOOLCHAIN)
224 toolchain.copy_files(resources.bin_files, BUILD_TOOLCHAIN)
226 objects = toolchain.compile_sources(resources, TMP_PATH)
227 toolchain.copy_files(objects, BUILD_TOOLCHAIN)
230 toolchain.info("Building library %s (%s, %s)" % ('MBED', target.name, toolchain_name))
233 toolchain.copy_files(toolchain.scan_resources(MBED_API).headers, MBED_LIBRARIES)
234 toolchain.copy_files(toolchain.scan_resources(MBED_HAL).headers, MBED_LIBRARIES)
236 # Target specific sources
237 HAL_SRC = join(MBED_TARGETS_PATH, "hal")
238 hal_implementation = toolchain.scan_resources(HAL_SRC)
239 toolchain.copy_files(hal_implementation.headers + hal_implementation.hex_files + hal_implementation.libraries, BUILD_TARGET, HAL_SRC)
240 incdirs = toolchain.scan_resources(BUILD_TARGET).inc_dirs
241 objects = toolchain.compile_sources(hal_implementation, TMP_PATH, [MBED_LIBRARIES] + incdirs)
244 mbed_resources = toolchain.scan_resources(MBED_COMMON)
245 objects += toolchain.compile_sources(mbed_resources, TMP_PATH, [MBED_LIBRARIES] + incdirs)
247 # A number of compiled files need to be copied as objects as opposed to
248 # being part of the mbed library, for reasons that have to do with the way
249 # the linker search for symbols in archives. These are:
250 # - retarget.o: to make sure that the C standard lib symbols get overridden
251 # - board.o: mbed_die is weak
252 # - mbed_overrides.o: this contains platform overrides of various weak SDK functions
253 separate_names, separate_objects = ['retarget.o', 'board.o', 'mbed_overrides.o'], []
255 for name in separate_names:
257 separate_objects.append(o)
258 for o in separate_objects:
260 toolchain.build_library(objects, BUILD_TOOLCHAIN, "mbed")
261 for o in separate_objects:
262 toolchain.copy_files(o, BUILD_TOOLCHAIN)
266 def get_unique_supported_toolchains():
267 """ Get list of all unique toolchains supported by targets """
268 unique_supported_toolchains = []
269 for target in TARGET_NAMES:
270 for toolchain in TARGET_MAP[target].supported_toolchains:
271 if toolchain not in unique_supported_toolchains:
272 unique_supported_toolchains.append(toolchain)
273 return unique_supported_toolchains
276 def mcu_toolchain_matrix(verbose_html=False, platform_filter=None):
277 """ Shows target map using prettytable """
278 unique_supported_toolchains = get_unique_supported_toolchains()
279 from prettytable import PrettyTable # Only use it in this function so building works without extra modules
281 # All tests status table print
282 columns = ["Platform"] + unique_supported_toolchains
283 pt = PrettyTable(["Platform"] + unique_supported_toolchains)
287 pt.align["Platform"] = "l"
291 for target in sorted(TARGET_NAMES):
292 if platform_filter is not None:
293 # FIlter out platforms using regex
294 if re.search(platform_filter, target) is None:
298 row = [target] # First column is platform name
299 default_toolchain = TARGET_MAP[target].default_toolchain
300 for unique_toolchain in unique_supported_toolchains:
302 if default_toolchain == unique_toolchain:
305 elif unique_toolchain in TARGET_MAP[target].supported_toolchains:
311 result = pt.get_html_string() if verbose_html else pt.get_string()
313 result += "*Default - default on-line compiler\n"
314 result += "*Supported - supported off-line compiler\n"
316 result += "Total platforms: %d\n"% (target_counter)
317 result += "Total permutations: %d"% (perm_counter)
321 def get_target_supported_toolchains(target):
322 """ Returns target supported toolchains list """
323 return TARGET_MAP[target].supported_toolchains if target in TARGET_MAP else None
326 def static_analysis_scan(target, toolchain_name, CPPCHECK_CMD, CPPCHECK_MSG_FORMAT, options=None, verbose=False, clean=False, macros=None, notify=None, jobs=1):
328 toolchain = TOOLCHAIN_CLASSES[toolchain_name](target, options, macros=macros, notify=notify)
329 toolchain.VERBOSE = verbose
330 toolchain.jobs = jobs
331 toolchain.build_all = clean
333 # Source and Build Paths
334 BUILD_TARGET = join(MBED_LIBRARIES, "TARGET_" + target.name)
335 BUILD_TOOLCHAIN = join(BUILD_TARGET, "TOOLCHAIN_" + toolchain.name)
336 mkdir(BUILD_TOOLCHAIN)
338 TMP_PATH = join(MBED_LIBRARIES, '.temp', toolchain.obj_path)
342 toolchain.info("Static analysis for %s (%s, %s)" % ('CMSIS', target.name, toolchain_name))
343 cmsis_src = join(MBED_TARGETS_PATH, "cmsis")
344 resources = toolchain.scan_resources(cmsis_src)
346 # Copy files before analysis
347 toolchain.copy_files(resources.headers, BUILD_TARGET)
348 toolchain.copy_files(resources.linker_script, BUILD_TOOLCHAIN)
350 # Gather include paths, c, cpp sources and macros to transfer to cppcheck command line
351 includes = ["-I%s"% i for i in resources.inc_dirs]
352 includes.append("-I%s"% str(BUILD_TARGET))
353 c_sources = " ".join(resources.c_sources)
354 cpp_sources = " ".join(resources.cpp_sources)
355 macros = ["-D%s"% s for s in toolchain.get_symbols() + toolchain.macros]
357 includes = map(str.strip, includes)
358 macros = map(str.strip, macros)
360 check_cmd = CPPCHECK_CMD
361 check_cmd += CPPCHECK_MSG_FORMAT
362 check_cmd += includes
365 # We need to pass some params via file to avoid "command line too long in some OSs"
366 tmp_file = tempfile.NamedTemporaryFile(delete=False)
367 tmp_file.writelines(line + '\n' for line in c_sources.split())
368 tmp_file.writelines(line + '\n' for line in cpp_sources.split())
370 check_cmd += ["--file-list=%s"% tmp_file.name]
372 _stdout, _stderr, _rc = run_cmd(check_cmd)
377 # =========================================================================
380 toolchain.info("Static analysis for %s (%s, %s)" % ('MBED', target.name, toolchain_name))
383 toolchain.copy_files(toolchain.scan_resources(MBED_API).headers, MBED_LIBRARIES)
384 toolchain.copy_files(toolchain.scan_resources(MBED_HAL).headers, MBED_LIBRARIES)
386 # Target specific sources
387 HAL_SRC = join(MBED_TARGETS_PATH, "hal")
388 hal_implementation = toolchain.scan_resources(HAL_SRC)
390 # Copy files before analysis
391 toolchain.copy_files(hal_implementation.headers + hal_implementation.hex_files, BUILD_TARGET, HAL_SRC)
392 incdirs = toolchain.scan_resources(BUILD_TARGET)
394 target_includes = ["-I%s" % i for i in incdirs.inc_dirs]
395 target_includes.append("-I%s"% str(BUILD_TARGET))
396 target_includes.append("-I%s"% str(HAL_SRC))
397 target_c_sources = " ".join(incdirs.c_sources)
398 target_cpp_sources = " ".join(incdirs.cpp_sources)
399 target_macros = ["-D%s"% s for s in toolchain.get_symbols() + toolchain.macros]
402 mbed_resources = toolchain.scan_resources(MBED_COMMON)
404 # Gather include paths, c, cpp sources and macros to transfer to cppcheck command line
405 mbed_includes = ["-I%s" % i for i in mbed_resources.inc_dirs]
406 mbed_includes.append("-I%s"% str(BUILD_TARGET))
407 mbed_includes.append("-I%s"% str(MBED_COMMON))
408 mbed_includes.append("-I%s"% str(MBED_API))
409 mbed_includes.append("-I%s"% str(MBED_HAL))
410 mbed_c_sources = " ".join(mbed_resources.c_sources)
411 mbed_cpp_sources = " ".join(mbed_resources.cpp_sources)
413 target_includes = map(str.strip, target_includes)
414 mbed_includes = map(str.strip, mbed_includes)
415 target_macros = map(str.strip, target_macros)
417 check_cmd = CPPCHECK_CMD
418 check_cmd += CPPCHECK_MSG_FORMAT
419 check_cmd += target_includes
420 check_cmd += mbed_includes
421 check_cmd += target_macros
423 # We need to pass some parames via file to avoid "command line too long in some OSs"
424 tmp_file = tempfile.NamedTemporaryFile(delete=False)
425 tmp_file.writelines(line + '\n' for line in target_c_sources.split())
426 tmp_file.writelines(line + '\n' for line in target_cpp_sources.split())
427 tmp_file.writelines(line + '\n' for line in mbed_c_sources.split())
428 tmp_file.writelines(line + '\n' for line in mbed_cpp_sources.split())
430 check_cmd += ["--file-list=%s"% tmp_file.name]
432 _stdout, _stderr, _rc = run_cmd_ext(check_cmd)
438 def static_analysis_scan_lib(lib_id, target, toolchain, cppcheck_cmd, cppcheck_msg_format,
439 options=None, verbose=False, clean=False, macros=None, notify=None, jobs=1):
440 lib = Library(lib_id)
441 if lib.is_supported(target, toolchain):
442 static_analysis_scan_library(lib.source_dir, lib.build_dir, target, toolchain, cppcheck_cmd, cppcheck_msg_format,
443 lib.dependencies, options,
444 verbose=verbose, clean=clean, macros=macros, notify=notify, jobs=jobs)
446 print 'Library "%s" is not yet supported on target %s with toolchain %s'% (lib_id, target.name, toolchain)
449 def static_analysis_scan_library(src_paths, build_path, target, toolchain_name, cppcheck_cmd, cppcheck_msg_format,
450 dependencies_paths=None, options=None, name=None, clean=False,
451 notify=None, verbose=False, macros=None, jobs=1):
452 """ Function scans library (or just some set of sources/headers) for staticly detectable defects """
453 if type(src_paths) != ListType:
454 src_paths = [src_paths]
456 for src_path in src_paths:
457 if not exists(src_path):
458 raise Exception("The library source folder does not exist: %s", src_path)
461 toolchain = TOOLCHAIN_CLASSES[toolchain_name](target, options, macros=macros, notify=notify)
462 toolchain.VERBOSE = verbose
463 toolchain.jobs = jobs
465 # The first path will give the name to the library
466 name = basename(src_paths[0])
467 toolchain.info("Static analysis for library %s (%s, %s)" % (name.upper(), target.name, toolchain_name))
471 for src_path in src_paths:
472 resources.append(toolchain.scan_resources(src_path))
474 # Dependencies Include Paths
475 dependencies_include_dir = []
476 if dependencies_paths is not None:
477 for path in dependencies_paths:
478 lib_resources = toolchain.scan_resources(path)
479 dependencies_include_dir.extend(lib_resources.inc_dirs)
481 # Create the desired build directory structure
482 bin_path = join(build_path, toolchain.obj_path)
484 tmp_path = join(build_path, '.temp', toolchain.obj_path)
487 # Gather include paths, c, cpp sources and macros to transfer to cppcheck command line
488 includes = ["-I%s" % i for i in dependencies_include_dir + src_paths]
491 macros = ['-D%s' % s for s in toolchain.get_symbols() + toolchain.macros]
494 for resource in resources:
495 toolchain.copy_files(resource.headers, build_path, rel_path=resource.base_path)
496 includes += ["-I%s" % i for i in resource.inc_dirs]
497 c_sources += " ".join(resource.c_sources) + " "
498 cpp_sources += " ".join(resource.cpp_sources) + " "
500 dependencies_include_dir.extend(toolchain.scan_resources(build_path).inc_dirs)
502 includes = map(str.strip, includes)
503 macros = map(str.strip, macros)
505 check_cmd = cppcheck_cmd
506 check_cmd += cppcheck_msg_format
507 check_cmd += includes
510 # We need to pass some parameters via file to avoid "command line too long in some OSs"
511 # Temporary file is created to store e.g. cppcheck list of files for command line
512 tmp_file = tempfile.NamedTemporaryFile(delete=False)
513 tmp_file.writelines(line + '\n' for line in c_sources.split())
514 tmp_file.writelines(line + '\n' for line in cpp_sources.split())
516 check_cmd += ["--file-list=%s"% tmp_file.name]
518 # This will allow us to grab result from both stdio and stderr outputs (so we can show them)
519 # We assume static code analysis tool is outputting defects on STDERR
520 _stdout, _stderr, _rc = run_cmd_ext(check_cmd)
526 def print_build_results(result_list, build_name):
527 """ Generate result string for build results """
530 result += build_name + "\n"
531 result += "\n".join([" * %s" % f for f in result_list])
535 def write_build_report(build_report, template_filename, filename):
536 build_report_failing = []
537 build_report_passing = []
539 for report in build_report:
540 if len(report["failing"]) > 0:
541 build_report_failing.append(report)
543 build_report_passing.append(report)
545 env = Environment(extensions=['jinja2.ext.with_'])
546 env.loader = FileSystemLoader('ci_templates')
547 template = env.get_template(template_filename)
549 with open(filename, 'w+') as f:
550 f.write(template.render(failing_builds=build_report_failing, passing_builds=build_report_passing))