v1.7.2

configure.py

@@ -98,7 +98,7 @@ def uses_usr_local(self):
  return self._platform in ('freebsd', 'openbsd', 'bitrig')
 
  def supports_ppoll(self):
- return self._platform in ('linux', 'openbsd', 'bitrig')
+ return self._platform in ('freebsd', 'linux', 'openbsd', 'bitrig')
 
  def supports_ninja_browse(self):
  return (not self.is_windows()

misc/measure.py

@@ -17,14 +17,16 @@
 """measure the runtime of a command by repeatedly running it.
 """
 
+from __future__ import print_function
+
 import time
 import subprocess
 import sys
 
 devnull = open('/dev/null', 'w')
 
 def run(cmd, repeat=10):
- print 'sampling:',
+ print('sampling:', end=' ')
  sys.stdout.flush()
 
  samples = []
@@ -33,10 +35,10 @@ def run(cmd, repeat=10):
  subprocess.call(cmd, stdout=devnull, stderr=devnull)
  end = time.time()
  dt = (end - start) * 1000
- print '%dms' % int(dt),
+ print('%dms' % int(dt), end=' ')
  sys.stdout.flush()
  samples.append(dt)
- print
+ print()
 
  # We're interested in the 'pure' runtime of the code, which is
  # conceptually the smallest time we'd see if we ran it enough times
@@ -45,10 +47,10 @@ def run(cmd, repeat=10):
  # Also print how varied the outputs were in an attempt to make it
  # more obvious if something has gone terribly wrong.
  err = sum(s - best for s in samples) / float(len(samples))
- print 'estimate: %dms (mean err %.1fms)' % (best, err)
+ print('estimate: %dms (mean err %.1fms)' % (best, err))
 
 if __name__ == '__main__':
  if len(sys.argv) < 2:
- print 'usage: measure.py command args...'
+ print('usage: measure.py command args...')
  sys.exit(1)
  run(cmd=sys.argv[1:])

misc/write_fake_manifests.py

@@ -50,9 +50,10 @@ def moar(avg_options, p_suffix):
 
 
 class GenRandom(object):
- def __init__(self):
+ def __init__(self, src_dir):
  self.seen_names = set([None])
  self.seen_defines = set([None])
+ self.src_dir = src_dir
 
  def _unique_string(self, seen, avg_options=1.3, p_suffix=0.1):
  s = None
@@ -76,7 +77,7 @@ def path(self):
 
  def src_obj_pairs(self, path, name):
  num_sources = paretoint(55, alpha=2) + 1
- return [(os.path.join('..', '..', path, s + '.cc'),
+ return [(os.path.join(self.src_dir, path, s + '.cc'),
  os.path.join('obj', path, '%s.%s.o' % (name, s)))
  for s in self._n_unique_strings(num_sources)]
 
@@ -103,12 +104,8 @@ def __init__(self, gen, kind):
  self.kind = kind
  self.has_compile_depends = random.random() < 0.4
 
- @property
- def includes(self):
- return ['-I' + dep.dir_path for dep in self.deps]
 
-
-def write_target_ninja(ninja, target):
+def write_target_ninja(ninja, target, src_dir):
  compile_depends = None
  if target.has_compile_depends:
  compile_depends = os.path.join(
@@ -117,8 +114,7 @@ def write_target_ninja(ninja, target):
  ninja.newline()
 
  ninja.variable('defines', target.defines)
- if target.deps:
- ninja.variable('includes', target.includes)
+ ninja.variable('includes', '-I' + src_dir)
  ninja.variable('cflags', ['-Wall', '-fno-rtti', '-fno-exceptions'])
  ninja.newline()
 
@@ -129,17 +125,63 @@ def write_target_ninja(ninja, target):
  deps = [dep.output for dep in target.deps]
  libs = [dep.output for dep in target.deps if dep.kind == LIB]
  if target.kind == EXE:
- ninja.variable('ldflags', '-Wl,pie')
  ninja.variable('libs', libs)
+ if sys.platform == "darwin":
+ ninja.variable('ldflags', '-Wl,-pie')
  link = { LIB: 'alink', EXE: 'link'}[target.kind]
  ninja.build(target.output, link, [obj for _, obj in target.src_obj_pairs],
  implicit=deps)
 
 
+def write_sources(target, root_dir):
+ need_main = target.kind == EXE
+
+ includes = []
+
+ # Include siblings.
+ for cc_filename, _ in target.src_obj_pairs:
+ h_filename = os.path.basename(os.path.splitext(cc_filename)[0] + '.h')
+ includes.append(h_filename)
+
+ # Include deps.
+ for dep in target.deps:
+ for cc_filename, _ in dep.src_obj_pairs:
+ h_filename = os.path.basename(
+ os.path.splitext(cc_filename)[0] + '.h')
+ includes.append("%s/%s" % (dep.dir_path, h_filename))
+
+ for cc_filename, _ in target.src_obj_pairs:
+ cc_path = os.path.join(root_dir, cc_filename)
+ h_path = os.path.splitext(cc_path)[0] + '.h'
+ namespace = os.path.basename(target.dir_path)
+ class_ = os.path.splitext(os.path.basename(cc_filename))[0]
+ try:
+ os.makedirs(os.path.dirname(cc_path))
+ except OSError:
+ pass
+
+ with open(h_path, 'w') as f:
+ f.write('namespace %s { struct %s { %s(); }; }' % (namespace,
+ class_, class_))
+ with open(cc_path, 'w') as f:
+ for include in includes:
+ f.write('#include "%s"\n' % include)
+ f.write('\n')
+ f.write('namespace %s { %s::%s() {} }' % (namespace,
+ class_, class_))
+
+ if need_main:
+ f.write('int main(int argc, char **argv) {}\n')
+ need_main = False
+
 def write_master_ninja(master_ninja, targets):
  """Writes master build.ninja file, referencing all given subninjas."""
  master_ninja.variable('cxx', 'c++')
  master_ninja.variable('ld', '$cxx')
+ if sys.platform == 'darwin':
+ master_ninja.variable('alink', 'libtool -static')
+ else:
+ master_ninja.variable('alink', 'ar rcs')
  master_ninja.newline()
 
  master_ninja.pool('link_pool', depth=4)
@@ -148,8 +190,8 @@ def write_master_ninja(master_ninja, targets):
  master_ninja.rule('cxx', description='CXX $out',
  command='$cxx -MMD -MF $out.d $defines $includes $cflags -c $in -o $out',
  depfile='$out.d', deps='gcc')
- master_ninja.rule('alink', description='LIBTOOL-STATIC $out',
- command='rm -f $out && libtool -static -o $out $in')
+ master_ninja.rule('alink', description='ARCHIVE $out',
+ command='rm -f $out && $alink -o $out $in')
  master_ninja.rule('link', description='LINK $out', pool='link_pool',
  command='$ld $ldflags -o $out $in $libs')
  master_ninja.rule('stamp', description='STAMP $out', command='touch $out')
@@ -181,9 +223,8 @@ def FileWriter(path):
  f.close()
 
 
-def random_targets():
- num_targets = 1500
- gen = GenRandom()
+def random_targets(num_targets, src_dir):
+ gen = GenRandom(src_dir)
 
  # N-1 static libraries, and 1 executable depending on all of them.
  targets = [Target(gen, LIB) for i in xrange(num_targets - 1)]
@@ -199,16 +240,28 @@ def random_targets():
 
 def main():
  parser = argparse.ArgumentParser()
+ parser.add_argument('-s', '--sources', nargs="?", const="src",
+ help='write sources to directory (relative to output directory)')
+ parser.add_argument('-t', '--targets', type=int, default=1500,
+ help='number of targets (default: 1500)')
+ parser.add_argument('-S', '--seed', type=int, help='random seed',
+ default=12345)
  parser.add_argument('outdir', help='output directory')
  args = parser.parse_args()
  root_dir = args.outdir
 
- random.seed(12345)
+ random.seed(args.seed)
 
- targets = random_targets()
+ do_write_sources = args.sources is not None
+ src_dir = args.sources if do_write_sources else "src"
+
+ targets = random_targets(args.targets, src_dir)
  for target in targets:
  with FileWriter(os.path.join(root_dir, target.ninja_file_path)) as n:
- write_target_ninja(n, target)
+ write_target_ninja(n, target, src_dir)
+
+ if do_write_sources:
+ write_sources(target, root_dir)
 
  with FileWriter(os.path.join(root_dir, 'build.ninja')) as master_ninja:
  master_ninja.width = 120

src/browse.py

@@ -27,6 +27,7 @@
 except ImportError:
  import BaseHTTPServer as httpserver
 import argparse
+import cgi
 import os
 import socket
 import subprocess
@@ -58,6 +59,9 @@ def match_strip(line, prefix):
  return (False, line)
  return (True, line[len(prefix):])
 
+def html_escape(text):
+ return cgi.escape(text, quote=True)
+
 def parse(text):
  lines = iter(text.split('\n'))
 
@@ -124,27 +128,27 @@ def create_page(body):
 ''' + body
 
 def generate_html(node):
- document = ['<h1><tt>%s</tt></h1>' % node.target]
+ document = ['<h1><tt>%s</tt></h1>' % html_escape(node.target)]
 
  if node.inputs:
  document.append('<h2>target is built using rule <tt>%s</tt> of</h2>' %
- node.rule)
+ html_escape(node.rule))
  if len(node.inputs) > 0:
  document.append('<div class=filelist>')
  for input, type in sorted(node.inputs):
  extra = ''
  if type:
- extra = ' (%s)' % type
+ extra = ' (%s)' % html_escape(type)
  document.append('<tt><a href="?%s">%s</a>%s</tt><br>' %
- (input, input, extra))
+ (html_escape(input), html_escape(input), extra))
  document.append('</div>')
 
  if node.outputs:
  document.append('<h2>dependent edges build:</h2>')
  document.append('<div class=filelist>')
  for output in sorted(node.outputs):
  document.append('<tt><a href="?%s">%s</a></tt><br>' %
- (output, output))
+ (html_escape(output), html_escape(output)))
  document.append('</div>')
 
  return '\n'.join(document)
@@ -177,7 +181,7 @@ def do_GET(self):
  page_body = generate_html(parse(ninja_output.strip()))
  else:
  # Relay ninja's error message.
- page_body = '<h1><tt>%s</tt></h1>' % ninja_error
+ page_body = '<h1><tt>%s</tt></h1>' % html_escape(ninja_error)
 
  self.send_response(200)
  self.end_headers()

src/build.cc

@@ -97,7 +97,7 @@ void BuildStatus::BuildEdgeStarted(Edge* edge) {
  ++started_edges_;
 
  if (edge->use_console() || printer_.is_smart_terminal())
- PrintStatus(edge);
+ PrintStatus(edge, kEdgeStarted);
 
  if (edge->use_console())
  printer_.SetConsoleLocked(true);
@@ -109,6 +109,7 @@ void BuildStatus::BuildEdgeFinished(Edge* edge,
  int* start_time,
  int* end_time) {
  int64_t now = GetTimeMillis();
+
  ++finished_edges_;
 
  RunningEdgeMap::iterator i = running_edges_.find(edge);
@@ -123,7 +124,7 @@ void BuildStatus::BuildEdgeFinished(Edge* edge,
  return;
 
  if (!edge->use_console())
- PrintStatus(edge);
+ PrintStatus(edge, kEdgeFinished);
 
  // Print the command that is spewing before printing its output.
  if (!success) {
@@ -158,13 +159,18 @@ void BuildStatus::BuildEdgeFinished(Edge* edge,
  }
 }
 
+void BuildStatus::BuildStarted() {
+ overall_rate_.Restart();
+ current_rate_.Restart();
+}
+
 void BuildStatus::BuildFinished() {
  printer_.SetConsoleLocked(false);
  printer_.PrintOnNewLine("");
 }
 
 string BuildStatus::FormatProgressStatus(
- const char* progress_status_format) const {
+ const char* progress_status_format, EdgeStatus status) const {
  string out;
  char buf[32];
  int percent;
@@ -189,10 +195,15 @@ string BuildStatus::FormatProgressStatus(
  break;
 
  // Running edges.
- case 'r':
- snprintf(buf, sizeof(buf), "%d", started_edges_ - finished_edges_);
+ case 'r': {
+ int running_edges = started_edges_ - finished_edges_;
+ // count the edge that just finished as a running edge
+ if (status == kEdgeFinished)
+ running_edges++;
+ snprintf(buf, sizeof(buf), "%d", running_edges);
  out += buf;
  break;
+ }
 
  // Unstarted edges.
  case 'u':
@@ -209,14 +220,14 @@ string BuildStatus::FormatProgressStatus(
  // Overall finished edges per second.
  case 'o':
  overall_rate_.UpdateRate(finished_edges_);
- snprinfRate(overall_rate_.rate(), buf, "%.1f");
+ SnprintfRate(overall_rate_.rate(), buf, "%.1f");
  out += buf;
  break;
 
  // Current rate, average over the last '-j' jobs.
  case 'c':
  current_rate_.UpdateRate(finished_edges_);
- snprinfRate(current_rate_.rate(), buf, "%.1f");
+ SnprintfRate(current_rate_.rate(), buf, "%.1f");
  out += buf;
  break;
 
@@ -246,7 +257,7 @@ string BuildStatus::FormatProgressStatus(
  return out;
 }
 
-void BuildStatus::PrintStatus(Edge* edge) {
+void BuildStatus::PrintStatus(Edge* edge, EdgeStatus status) {
  if (config_.verbosity == BuildConfig::QUIET)
  return;
 
@@ -256,11 +267,7 @@ void BuildStatus::PrintStatus(Edge* edge) {
  if (to_print.empty() || force_full_command)
  to_print = edge->GetBinding("command");
 
- if (finished_edges_ == 0) {
- overall_rate_.Restart();
- current_rate_.Restart();
- }
- to_print = FormatProgressStatus(progress_status_format_) + to_print;
+ to_print = FormatProgressStatus(progress_status_format_, status) + to_print;
 
  printer_.Print(to_print,
  force_full_command ? LinePrinter::FULL : LinePrinter::ELIDE);
@@ -643,6 +650,9 @@ bool Builder::Build(string* err) {
  command_runner_.reset(new RealCommandRunner(config_));
  }
 
+ // We are about to start the build process.
+ status_->BuildStarted();
+
  // This main loop runs the entire build process.
  // It is structured like this:
  // First, we attempt to start as many commands as allowed by the