3 Copyright (c) 2011-2013 ARM Limited
5 Licensed under the Apache License, Version 2.0 (the "License");
6 you may not use this file except in compliance with the License.
7 You may obtain a copy of the License at
9 http://www.apache.org/licenses/LICENSE-2.0
11 Unless required by applicable law or agreed to in writing, software
12 distributed under the License is distributed on an "AS IS" BASIS,
13 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 See the License for the specific language governing permissions and
15 limitations under the License.
18 One repository to update them all
19 On mbed.org the mbed SDK is split up in multiple repositories, this script takes
20 care of updating them all.
24 from os import walk, remove, makedirs
25 from os.path import join, abspath, dirname, relpath, exists, isfile
26 from shutil import copyfile
27 from optparse import OptionParser
31 ROOT = abspath(join(dirname(__file__), ".."))
32 sys.path.insert(0, ROOT)
34 from workspace_tools.settings import MBED_ORG_PATH, MBED_ORG_USER, BUILD_DIR
35 from workspace_tools.paths import LIB_DIR
36 from workspace_tools.utils import run_cmd
39 MBED_USER = "mbed_official"
46 # Code that does have a mirror in the mbed SDK
47 # Tuple data: (repo_name, list_of_code_dirs, [team])
48 # team is optional - if not specified, the code is published under mbed_official
50 ("mbed-src" , "mbed"),
51 ("mbed-rtos", "rtos"),
55 ("lwip" , "net/lwip/lwip"),
56 ("lwip-sys", "net/lwip/lwip-sys"),
57 ("Socket" , "net/lwip/Socket"),
59 ("lwip-eth" , "net/eth/lwip-eth"),
60 ("EthernetInterface", "net/eth/EthernetInterface"),
62 ("USBDevice", "USBDevice"),
63 ("USBHost" , "USBHost"),
65 ("CellularModem", "net/cellular/CellularModem"),
66 ("CellularUSBModem", "net/cellular/CellularUSBModem"),
67 ("UbloxUSBModem", "net/cellular/UbloxUSBModem"),
68 ("UbloxModemHTTPClientTest", ["tests/net/cellular/http/common", "tests/net/cellular/http/ubloxusb"]),
69 ("UbloxModemSMSTest", ["tests/net/cellular/sms/common", "tests/net/cellular/sms/ubloxusb"]),
70 ("FATFileSystem", "fs/fat", "mbed-official"),
74 # Code that does have dependencies to libraries should point to
75 # the latest revision. By default, they point to a specific revision.
76 CODE_WITH_DEPENDENCIES = (
93 "TCPSocket_HelloWorld",
94 "UDPSocket_HelloWorld",
104 # A list of regular expressions that will be checked against each directory
105 # name and skipped if they match.
116 def ignore_path(name, reg_exps):
118 if re.search(r, name):
122 class MbedRepository:
124 def run_and_print(command, cwd):
125 stdout, _, _ = run_cmd(command, wd=cwd, redirect=True)
128 def __init__(self, name, team = None):
130 self.path = join(MBED_ORG_PATH, name)
132 self.url = "http://" + MBED_URL + "/users/" + MBED_USER + "/code/%s/"
134 self.url = "http://" + MBED_URL + "/teams/" + team + "/code/%s/"
135 if not exists(self.path):
137 if not exists(MBED_ORG_PATH):
138 makedirs(MBED_ORG_PATH)
140 self.run_and_print(['hg', 'clone', self.url % name], cwd=MBED_ORG_PATH)
144 self.run_and_print(['hg', 'pull'], cwd=self.path)
145 self.run_and_print(['hg', 'update'], cwd=self.path)
148 # The maintainer has to evaluate the changes first and explicitly accept them
149 self.run_and_print(['hg', 'addremove'], cwd=self.path)
150 stdout, _, _ = run_cmd(['hg', 'status'], wd=self.path)
158 commit = raw_input(push_remote and "Do you want to commit and push? Y/N: " or "Do you want to commit? Y/N: ")
160 args = ['hg', 'commit', '-u', MBED_ORG_USER]
162 args = args + ['-m', commit_msg]
163 self.run_and_print(args, cwd=self.path)
165 self.run_and_print(['hg', 'push'], cwd=self.path)
168 # Check if a file is a text file or a binary file
169 # Taken from http://code.activestate.com/recipes/173220/
170 text_characters = "".join(map(chr, range(32, 127)) + list("\n\r\t\b"))
171 _null_trans = string.maketrans("", "")
172 def is_text_file(filename):
178 if not s: # Empty files are considered text
181 # Get the non-text characters (maps a character to itself then
182 # use the 'remove' option to get rid of the text characters.)
183 t = s.translate(_null_trans, text_characters)
185 # If more than 30% non-text characters, then
186 # this is considered a binary file
187 if float(len(t))/len(s) > 0.30:
190 with open(filename) as f:
191 res = istext(f.read(block_size))
194 # Return the line ending type for the given file ('cr' or 'crlf')
195 def get_line_endings(f):
199 lines, ncrlf = tf.readlines(examine_size), 0
202 if l.endswith("\r\n"):
204 return 'crlf' if ncrlf > len(lines) >> 1 else 'cr'
208 # Copy file to destination, but preserve destination line endings if possible
209 # This prevents very annoying issues with huge diffs that appear because of
210 # differences in line endings
211 def copy_with_line_endings(sdk_file, repo_file):
212 if not isfile(repo_file):
213 copyfile(sdk_file, repo_file)
215 is_text = is_text_file(repo_file)
217 sdk_le = get_line_endings(sdk_file)
218 repo_le = get_line_endings(repo_file)
219 if not is_text or sdk_le == repo_le:
220 copyfile(sdk_file, repo_file)
222 print "Converting line endings in '%s' to '%s'" % (abspath(repo_file), repo_le)
223 f = open(sdk_file, "rb")
226 f = open(repo_file, "wb")
227 data = data.replace("\r\n", "\n") if repo_le == 'cr' else data.replace('\n','\r\n')
231 def visit_files(path, visit):
232 for root, dirs, files in walk(path):
233 # Ignore hidden directories
236 if d.startswith('.'):
238 if ignore_path(full, IGNORE_DIRS):
239 print "Skipping '%s'" % full
243 if ignore_path(file, IGNORE_FILES):
246 visit(join(root, file))
249 def update_repo(repo_name, sdk_paths, team_name):
250 repo = MbedRepository(repo_name, team_name)
251 # copy files from mbed SDK to mbed_official repository
252 def visit_mbed_sdk(sdk_file):
253 repo_file = join(repo.path, relpath(sdk_file, sdk_path))
255 repo_dir = dirname(repo_file)
256 if not exists(repo_dir):
259 copy_with_line_endings(sdk_file, repo_file)
260 for sdk_path in sdk_paths:
261 visit_files(sdk_path, visit_mbed_sdk)
263 # remove repository files that do not exist in the mbed SDK
264 def visit_repo(repo_file):
265 for sdk_path in sdk_paths:
266 sdk_file = join(sdk_path, relpath(repo_file, repo.path))
271 print "remove: %s" % repo_file
272 visit_files(repo.path, visit_repo)
275 changed.append(repo_name)
278 def update_code(repositories):
279 for r in repositories:
280 repo_name, sdk_dir = r[0], r[1]
281 team_name = r[2] if len(r) == 3 else None
282 print '\n=== Updating "%s" ===' % repo_name
283 sdk_dirs = [sdk_dir] if type(sdk_dir) != type([]) else sdk_dir
284 sdk_path = [join(LIB_DIR, d) for d in sdk_dirs]
285 update_repo(repo_name, sdk_path, team_name)
287 def update_single_repo(repo):
288 repos = [r for r in OFFICIAL_CODE if r[0] == repo]
290 print "Repository '%s' not found" % repo
294 def update_dependencies(repositories):
295 for repo_name in repositories:
296 print '\n=== Updating "%s" ===' % repo_name
297 repo = MbedRepository(repo_name)
299 # point to the latest libraries
300 def visit_repo(repo_file):
301 with open(repo_file, "r") as f:
303 with open(repo_file, "w") as f:
304 f.write(url[:(url.rindex('/')+1)])
305 visit_files(repo.path, visit_repo, None, MBED_REPO_EXT)
308 changed.append(repo_name)
312 update_repo("mbed", [join(BUILD_DIR, "mbed")], None)
314 def do_sync(options):
315 global push_remote, quiet, commit_msg, changed
317 push_remote = not options.nopush
318 quiet = options.quiet
319 commit_msg = options.msg
323 update_code(OFFICIAL_CODE)
325 if options.dependencies:
326 update_dependencies(CODE_WITH_DEPENDENCIES)
332 update_single_repo(options.repo)
335 print "Repositories with changes:", changed
339 if __name__ == '__main__':
340 parser = OptionParser()
342 parser.add_option("-c", "--code",
343 action="store_true", default=False,
344 help="Update the mbed_official code")
346 parser.add_option("-d", "--dependencies",
347 action="store_true", default=False,
348 help="Update the mbed_official code dependencies")
350 parser.add_option("-m", "--mbed",
351 action="store_true", default=False,
352 help="Release a build of the mbed library")
354 parser.add_option("-n", "--nopush",
355 action="store_true", default=False,
356 help="Commit the changes locally only, don't push them")
358 parser.add_option("", "--commit_message",
359 action="store", type="string", default='', dest='msg',
360 help="Commit message to use for all the commits")
362 parser.add_option("-r", "--repository",
363 action="store", type="string", default='', dest='repo',
364 help="Synchronize only the given repository")
366 parser.add_option("-q", "--quiet",
367 action="store_true", default=False,
368 help="Don't ask for confirmation before commiting or pushing")
370 (options, args) = parser.parse_args()