diff --git a/test/problems b/test/problems
index dff55f2c1..b4f2ff65d 100755
--- a/test/problems
+++ b/test/problems
@@ -127,3 +127,7 @@ if __name__ == "__main__":
         print()
         print(color(expected_str, "yellow"))
         print_category(expected)
+
+    # If we encoutered any failures, return non-zero code
+    if sum(errors.values()):
+        sys.exit(1)
diff --git a/test/run_all b/test/run_all
index 29315dfb6..3c5914c71 100755
--- a/test/run_all
+++ b/test/run_all
@@ -177,7 +177,7 @@ class TestRunner(object):
         self.tap.flush()
 
         log.debug("Calling 'problems --summary' for report")
-        call([os.path.abspath("problems"), "--summary", cmd_args.tapfile])
+        return call([os.path.abspath("problems"), "--summary", cmd_args.tapfile])
 
 
 def parse_args():
@@ -197,8 +197,12 @@ def parse_args():
 def main():
     runner = TestRunner()
     runner.start()
+
+    # If we're producing summary report, propagate the return code
     if not cmd_args.verbose:
-        runner.show_report()
+        return runner.show_report()
+    else:
+        return 0
 
 
 if __name__ == "__main__":
@@ -222,7 +226,7 @@ if __name__ == "__main__":
     log.debug("Parsed commandline arguments: %s", cmd_args)
 
     try:
-        main()
+        sys.exit(main())
     except Exception as e:
         log.exception(e)
         sys.exit(1)