diff --git a/Gemfile.lock b/Gemfile.lock index 6637957..b7818f7 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,29 +1,33 @@ PATH remote: . specs: - bwoken (2.1.0.rc.2) + bwoken (2.1.0.rc.4) coffee-script-source colorful execjs json_pure + nokogiri (~> 1.6.5) rake - slop + slop (~> 3.6.0) GEM remote: https://rubygems.org/ specs: - coffee-script-source (1.6.3) + coffee-script-source (1.9.1) colorful (0.0.3) diff-lcs (1.2.1) - execjs (2.0.1) + execjs (2.3.0) ffi (1.0.11) guard (1.0.1) ffi (>= 0.5.0) thor (~> 0.14.6) guard-rspec (0.6.0) guard (>= 0.10.0) - json_pure (1.8.0) - rake (10.1.0) + json_pure (1.8.2) + mini_portile (0.6.2) + nokogiri (1.6.6.2) + mini_portile (~> 0.6.0) + rake (10.4.2) rspec (2.13.0) rspec-core (~> 2.13.0) rspec-expectations (~> 2.13.0) @@ -32,7 +36,7 @@ GEM rspec-expectations (2.13.0) diff-lcs (>= 1.1.3, < 2.0) rspec-mocks (2.13.0) - slop (3.4.6) + slop (3.6.0) thor (0.14.6) PLATFORMS diff --git a/README.md b/README.md index cc13bea..10a003f 100644 --- a/README.md +++ b/README.md @@ -116,6 +116,7 @@ $ bwoken test -h --configuration The build configruation to use (e.g., --configuration=Release) --sdk-version The SDK version to use (e.g., --sdk-version=6.1) --verbose Be verbose + --junit To output results in junit xml format -h, --help Display this help message. diff --git a/bin/unix_instruments.sh b/bin/unix_instruments.sh index 00a68a2..a208721 100755 --- a/bin/unix_instruments.sh +++ b/bin/unix_instruments.sh @@ -38,25 +38,10 @@ set -e # Bomb on any script errors run_instruments() { - # Because instruments buffers it's output if it determines that it is being - # piped to another process, we have to use ptys to get around that so that we - # can use `tee` to save the output for grepping and print to stdout in real - # time at the same time. - # - # I don't like this because I'm hard coding a tty/pty pair in here. Suggestions - # to make this cleaner? - + # Pipe to `tee` using a temporary file so everything is sent to standard out + # and we have the output to check for errors. output=$(mktemp -t unix-instruments) - instruments "$@" &> /dev/ttyvf & - pid_instruments=$! - - # Cat the instruments output to tee which outputs to stdout and saves to - # $output at the same time - cat < /dev/ptyvf | tee $output - - # Clear the process id we saved when forking instruments so the cleanup - # function called on exit knows it doesn't have to kill anything - pid_instruments=0 + instruments "$@" 2>&1 | tee $output # Process the instruments output looking for anything that resembles a fail # message @@ -72,16 +57,6 @@ get_error_status() { ruby -e 'exit 1 if STDIN.read =~ /Instruments Usage Error|Instruments Trace Error|^\d+-\d+-\d+ \d+:\d+:\d+ [-+]\d+ (Fail:|Error:|None: Script threw an uncaught JavaScript error)/' } -trap cleanup_instruments EXIT -cleanup_instruments() { - # Because we fork instruments in this script, we need to clean up if it's - # still running because of an error or the user pressed Ctrl-C - if [[ $pid_instruments -gt 0 ]]; then - echo "Cleaning up instruments..." - kill -9 $pid_instruments - fi -} - # Running this file with "----test" will try to parse an error out of whatever # is handed in to it from stdin. Use this method to double check your work if # you need a custom "get_error_status" function above. diff --git a/bwoken.gemspec b/bwoken.gemspec index 06a3c42..3a1881b 100644 --- a/bwoken.gemspec +++ b/bwoken.gemspec @@ -23,7 +23,8 @@ Gem::Specification.new do |gem| gem.add_dependency 'execjs' gem.add_dependency 'json_pure' gem.add_dependency 'rake' - gem.add_dependency 'slop' + gem.add_dependency 'slop', '~> 3.6.0' + gem.add_dependency 'nokogiri', '~> 1.6.5' gem.add_development_dependency 'rspec' gem.add_development_dependency 'guard-rspec' diff --git a/lib/bwoken/cli.rb b/lib/bwoken/cli.rb index 3988f0d..3e157a7 100644 --- a/lib/bwoken/cli.rb +++ b/lib/bwoken/cli.rb @@ -38,6 +38,7 @@ on :configuration=, 'The build configruation to use (e.g., --configuration=Release)', :default => 'Debug' on :'sdk-version=', 'The SDK version to use (e.g., --sdk-version=6.1)' on :verbose, 'Be verbose' + on :junit, 'Create junit xml test results' run { ran_command = 'test' } end diff --git a/lib/bwoken/cli/test.rb b/lib/bwoken/cli/test.rb index 6e55be6..f07a06d 100644 --- a/lib/bwoken/cli/test.rb +++ b/lib/bwoken/cli/test.rb @@ -8,8 +8,10 @@ require 'bwoken/device' #TODO: make formatters dynamically loadable during runtime require 'bwoken/formatter' +require 'bwoken/formatters/fanout_formatter' require 'bwoken/formatters/passthru_formatter' require 'bwoken/formatters/colorful_formatter' +require 'bwoken/formatters/junit_formatter' require 'bwoken/script_runner' module Bwoken @@ -43,6 +45,7 @@ def self.help_banner # :flags - custom build flag array (default: []) TODO: not yet implmented # :focus - which tests to run (default: [], meaning "all") # :formatter - custom formatter (default: 'colorful') + # :junit - create junit xml test results # :scheme - custom scheme (default: nil) # :simulator - should force simulator use (default: nil) # :skip-build - do not build the iOS binary @@ -54,8 +57,12 @@ def self.help_banner def initialize opts opts = opts.to_hash if opts.is_a?(Slop) self.options = opts.to_hash.tap do |o| - o[:formatter] = 'passthru' if o[:verbose] - o[:formatter] = select_formatter(o[:formatter]) + output_format_type = o[:verbose] ? 'passthru' : o[:formatter] + formatter = Bwoken::FanoutFormatter.new + formatter.add_recipient(select_formatter(output_format_type)) + formatter.add_recipient(Bwoken::JUnitFormatter.new) if o[:junit] + + o[:formatter] = formatter o[:simulator] = use_simulator?(o[:simulator]) o[:family] = o[:family] end diff --git a/lib/bwoken/formatter.rb b/lib/bwoken/formatter.rb index ff3a231..e7f1a2d 100644 --- a/lib/bwoken/formatter.rb +++ b/lib/bwoken/formatter.rb @@ -2,17 +2,9 @@ module Bwoken class Formatter class << self - def format stdout - new.format stdout - end - - def format_build stdout - new.format_build stdout - end - def on name, &block define_method "_on_#{name}_callback" do |*line| - block.call(*line) + instance_exec(*line, &block) end end @@ -25,32 +17,6 @@ def method_missing(method_name, *args, &block) end end - def line_demuxer line, exit_status - if line =~ /Instruments Trace Error/ - exit_status = 1 - _on_fail_callback(line) - elsif line =~ /^\d{4}/ - tokens = line.split(' ') - - if tokens[3] =~ /Pass/ - _on_pass_callback(line) - elsif tokens[3] =~ /Start/ - _on_start_callback(line) - elsif tokens[3] =~ /Fail/ || line =~ /Script threw an uncaught JavaScript error/ - exit_status = 1 - _on_fail_callback(line) - elsif tokens[3] =~ /Error/ - _on_error_callback(line) - else - _on_debug_callback(line) - end - elsif line =~ /Instruments Trace Complete/ - _on_complete_callback(line) - else - _on_other_callback(line) - end - exit_status - end %w(pass fail debug other).each do |log_level| on log_level.to_sym do |line| @@ -58,48 +24,5 @@ def line_demuxer line, exit_status end end - def format stdout - exit_status = 0 - - stdout.each_line do |line| - exit_status = line_demuxer line, exit_status - end - - exit_status - end - - def format_build stdout - out_string = '' - stdout.each_line do |line| - out_string << line - if line.length > 1 - _on_build_line_callback(line) - end - end - out_string - end - - on :before_build_start do - puts 'Building' - end - - on :build_line do |line| - print '.' - end - - on :build_successful do |build_log| - puts - puts - puts "### Build Successful ###" - puts - end - - on :build_failed do |build_log, error_log| - puts build_log - puts "Standard Error:" - puts error_log - puts '## Build failed ##' - end - end end diff --git a/lib/bwoken/formatters/fanout_formatter.rb b/lib/bwoken/formatters/fanout_formatter.rb new file mode 100644 index 0000000..c2c2ae0 --- /dev/null +++ b/lib/bwoken/formatters/fanout_formatter.rb @@ -0,0 +1,72 @@ +require 'bwoken/formatters/line_demuxer' +module Bwoken + + # Forwards any messages sent to this object to all recipients + # that respond to that message. + class FanoutFormatter < BasicObject + attr_reader :recipients + attr_reader :line_demuxer + + def initialize(line_demuxer = LineDemuxer.new) + @recipients = [] + @line_demuxer = line_demuxer + end + + def add_recipient(recipient) + recipients << recipient + end + + def format stdout + exit_status = 0 + + stdout.each_line do |line| + exit_status = @line_demuxer.demux(line, exit_status, recipients) + end + + exit_status + end + + def format_build stdout + out_string = '' + stdout.each_line do |line| + out_string << line + if line.length > 1 + send_to_recipients('_on_build_line_callback', line) + end + end + out_string + end + + def before_script_run(path) + send_to_recipients('_on_before_script_run_callback', path) + end + + def after_script_run + send_to_recipients('_on_after_script_run_callback') + end + + def before_build_start + send_to_recipients('_on_before_build_start_callback') + end + + def build_successful(line) + send_to_recipients('_on_build_successful_callback', line) + end + + def build_failed(build_log, error_log) + send_to_recipients('_on_build_failed_callback', *[build_log, error_log]) + end + + def send_to_recipients(message, *line) + message = message.to_sym + recipients.each do |recipient| + recipient.send(message, *line) if recipient.respond_to?(message) + end + end + + def to_s + + end + end + +end diff --git a/lib/bwoken/formatters/junit_formatter.rb b/lib/bwoken/formatters/junit_formatter.rb new file mode 100644 index 0000000..da46801 --- /dev/null +++ b/lib/bwoken/formatters/junit_formatter.rb @@ -0,0 +1,223 @@ +require 'nokogiri' +require 'bwoken/formatter' + +module Bwoken + + class JUnitTestSuite + attr_accessor :id + attr_accessor :package + attr_accessor :host_name + attr_accessor :name + attr_accessor :tests + attr_accessor :failures + attr_accessor :errors + attr_accessor :time + attr_accessor :timestamp + + attr_accessor :test_cases + + def initialize + self.test_cases = [] + self.tests = 0 + self.failures = 0 + self.errors = 0 + end + + def complete + self.time = Time.now - self.timestamp + end + + end + + class JUnitTestCase + attr_accessor :name + attr_accessor :classname + attr_accessor :time + attr_accessor :error + attr_accessor :logs + + attr_accessor :start_time + + def initialize + self.logs = String.new + self.error = nil + end + + def complete + self.time = Time.now - self.start_time + end + + end + + class JUnitFormatter < Formatter + attr_accessor :test_suites + + def initialize + self.test_suites = [] + end + + on :after_script_run do + generate_report + end + + on :complete do |line| + tokens = line.split(' ') + test_suite = self.test_suites.last + test_suite.time = tokens[5].sub(';', '') + end + + on :debug do |line| + filtered_line = line.sub(/(target\.frontMostApp.+)\.tap\(\)/, "#{'tap'} \\1") + filtered_line = filtered_line.gsub(/\[("[^\]]*")\]/, "[" + '\1' + "]") + filtered_line = filtered_line.gsub('()', '') + filtered_line = filtered_line.sub(/target.frontMostApp.(?:mainWindow.)?/, '') + tokens = filtered_line.split(' ') + + test_suite = self.test_suites.last + test_case = test_suite.test_cases.last + + if test_case + test_case.logs << "\n#{tokens[3].cyan}\t#{tokens[4..-1].join(' ')}" + end + end + + on :error do |line| + @failed = true + tokens = line.split(' ') + + test_suite = self.test_suites.last + test_case = test_suite.test_cases.last + if test_case + test_case.complete + test_case.error = tokens[4..-1].join(' ') + end + + test_suite.errors += 1 + + end + + on :fail do |line| + @failed = true + tokens = line.split(' ') + + test_suite = self.test_suites.last + test_case = test_suite.test_cases.last + if test_case + test_case.complete + test_case.error = tokens[4..-1].join(' ') + end + + test_suite.failures += 1 + + end + + on :start do |line| + tokens = line.split(' ') + + suite = self.test_suites.last + if suite + test_case = Bwoken::JUnitTestCase.new + test_case.name = tokens[4..-1].join(' ') + test_case.classname = test_case.name + test_case.start_time = Time.now + + suite.tests+=1 + suite.test_cases << test_case + end + end + + on :pass do |line| + tokens = line.split(' ') + + test_case = self.test_suites.last.test_cases.last + if test_case + test_case.complete + test_case.error = nil + end + end + + on :before_script_run do |path| + tokens = path.split('/') + + new_suite = Bwoken::JUnitTestSuite.new + new_suite.timestamp = Time.now + new_suite.host_name = tokens[-2] + new_suite.name = tokens[-1] + new_suite.package = new_suite.name + new_suite.id = self.test_suites.count + 1 + + self.test_suites << new_suite + + @failed = false + end + + on :other do |line| + nil + end + + def generate_report + doc = Nokogiri::XML::Document.new() + root = Nokogiri::XML::Element.new('testsuites', doc) + doc.add_child(root) + + result_name = 'unknown' + + self.test_suites.each do |suite| + result_name = suite.name.gsub /\.js$/, '' + + suite_elm = Nokogiri::XML::Element.new('testsuite', doc) + suite_elm['id'] = suite.id + suite_elm['package'] = suite.package + suite_elm['hostname'] = suite.host_name + suite_elm['name'] = suite.name + suite_elm['tests'] = suite.tests + suite_elm['failures'] = suite.failures + suite_elm['errors'] = suite.errors + suite_elm['time'] = suite.time + suite_elm['timestamp'] = suite.timestamp.to_s + + system_out = '' + system_err = '' + + suite.test_cases.each do |test_case| + test_case_elm = Nokogiri::XML::Element.new('testcase', doc) + test_case_elm['name'] = test_case.name + test_case_elm['classname'] = test_case.classname + test_case_elm['time'] = test_case.time + + if test_case.error + error = Nokogiri::XML::Element.new('error', doc) + error['type'] = test_case.error + test_case_elm.add_child(error) + system_err << "\n\n#{test_case.logs}" + else + system_out << "\n\n#{test_case.logs}" + end + + suite_elm.add_child(test_case_elm) + end + + suite_elm.add_child("#{doc.create_cdata(system_out)}") + suite_elm.add_child("#{doc.create_cdata(system_err)}") + + root.add_child(suite_elm) + + end + + out_xml = doc.to_xml + + write_results(out_xml, result_name) + end + + def write_results(xml, suite_name) + output_path = File.join(Bwoken.results_path, "#{suite_name}_results.xml") + File.open(output_path, 'w+') do |io| + io.write(xml) + end + + puts "\nJUnit report generated to #{output_path}\n\n" + end + + + end +end diff --git a/lib/bwoken/formatters/line_demuxer.rb b/lib/bwoken/formatters/line_demuxer.rb new file mode 100644 index 0000000..78439fb --- /dev/null +++ b/lib/bwoken/formatters/line_demuxer.rb @@ -0,0 +1,41 @@ +module Bwoken + class LineDemuxer + + def demux(line, exit_status, recipients) + if line =~ /Instruments Trace Error/ + exit_status = 1 + message = '_on_fail_callback' + #_on_fail_callback(line) + elsif line =~ /^\d{4}/ + tokens = line.split(' ') + + if tokens[3] =~ /Pass/ + message = '_on_pass_callback' + elsif tokens[3] =~ /Start/ + message = '_on_start_callback' + elsif tokens[3] =~ /Fail/ || line =~ /Script threw an uncaught JavaScript error/ + exit_status = 1 + message = '_on_fail_callback' + elsif tokens[3] =~ /Error/ + message = '_on_error_callback' + else + message = '_on_debug_callback' + end + elsif line =~ /Instruments Trace Complete/ + message = '_on_complete_callback' + else + message = '_on_other_callback' + end + + send_to_recipients(recipients, message, line) + exit_status + end + + def send_to_recipients(recipients, message, line) + message = message.to_sym + recipients.each do |recipient| + recipient.send(message, *line) if recipient.respond_to?(message) + end + end + end +end diff --git a/lib/bwoken/script.rb b/lib/bwoken/script.rb index 96d8c1f..a7e2f6a 100644 --- a/lib/bwoken/script.rb +++ b/lib/bwoken/script.rb @@ -48,18 +48,19 @@ def device_flag if !device.nil? return "-w \"#{device}\"" end - + simulator ? '' : "-w #{Bwoken::Device.uuid}" end def run formatter.before_script_run path - + exit_status = 0 Open3.popen3(cmd) do |stdin, stdout, stderr, wait_thr| exit_status = formatter.format stdout - raise ScriptFailedError.new('Test Script Failed') unless exit_status == 0 + break unless exit_status == 0 end + formatter.after_script_run + raise ScriptFailedError.new('Test Script Failed') unless exit_status == 0 end - end end diff --git a/lib/bwoken/version.rb b/lib/bwoken/version.rb index 7df0ee9..79cf940 100644 --- a/lib/bwoken/version.rb +++ b/lib/bwoken/version.rb @@ -1,3 +1,3 @@ module Bwoken - VERSION = "2.1.0.rc.2" unless defined?(::Bwoken::VERSION) + VERSION = "2.1.0.rc.4" unless defined?(::Bwoken::VERSION) end diff --git a/spec/lib/bwoken/cli/test_spec.rb b/spec/lib/bwoken/cli/test_spec.rb new file mode 100644 index 0000000..1c85a75 --- /dev/null +++ b/spec/lib/bwoken/cli/test_spec.rb @@ -0,0 +1,65 @@ +require 'spec_helper' + +require 'bwoken/cli/test' + +describe Bwoken::CLI::Test do + + describe '#init' do + let(:options) { { simulator: true } } + + subject { described_class.new(options) } + context 'formatters' do + + context 'when verbose' do + before do + options[:verbose] = true + end + + it 'should use passthru' do + expect(formatters.length).to be(1) + expect(formatters.first).to be_kind_of(Bwoken::PassthruFormatter) + end + end + + context 'not verbose' do + context 'when colorful' do + before do + options[:formatter] = 'colorful' + end + + it 'should use colorful' do + expect(formatters.length).to be(1) + expect(formatters.first).to be_kind_of(Bwoken::ColorfulFormatter) + end + end + + context 'when passthru' do + before do + options[:formatter] = 'passthru' + end + + it 'should use passthru' do + expect(formatters.length).to be(1) + expect(formatters.first).to be_kind_of(Bwoken::PassthruFormatter) + end + end + end + + context 'when junit' do + before do + options[:junit] = true + end + + it 'should use passthru' do + expect(formatters.length).to be(2) + expect(formatters.last).to be_kind_of(Bwoken::JUnitFormatter) + end + end + end + end + + def formatters + subject.options[:formatter].recipients + end + +end diff --git a/spec/lib/bwoken/formatter_spec.rb b/spec/lib/bwoken/formatter_spec.rb index ee13c19..984e866 100644 --- a/spec/lib/bwoken/formatter_spec.rb +++ b/spec/lib/bwoken/formatter_spec.rb @@ -2,14 +2,6 @@ require 'bwoken/formatter' describe Bwoken::Formatter do - describe '.format' do - it 'calls format on a new instance' do - formatter_stub = double('formatter') - formatter_stub.should_receive(:format).with('foo') - Bwoken::Formatter.stub(:new => formatter_stub) - Bwoken::Formatter.format 'foo' - end - end describe '.on' do let(:klass) { klass = Class.new(Bwoken::Formatter) } @@ -36,137 +28,4 @@ end end - describe '#line_demuxer' do - - context 'for a passing line' do - it 'calls _on_pass_callback' do - subject.should_receive(:_on_pass_callback).with('1234 a a Pass') - subject.line_demuxer('1234 a a Pass', 0) - end - it 'returns 0' do - exit_status = 0 - capture_stdout do - exit_status = subject.line_demuxer('1234 a a Pass', 0) - end - exit_status.should == 0 - end - end - - context 'for a failing line' do - context 'Fail error' do - it 'calls _on_fail_callback' do - subject.should_receive(:_on_fail_callback).with('1234 a a Fail') - subject.line_demuxer('1234 a a Fail', 0) - end - end - - context 'Instruments Trace Error message' do - it 'calls _on_fail_callback' do - msg = 'Instruments Trace Error foo' - subject.should_receive(:_on_fail_callback).with(msg) - subject.line_demuxer(msg, 0) - end - end - - it 'returns 1' do - exit_status = 0 - capture_stdout do - exit_status = subject.line_demuxer('1234 a a Fail', 0) - end - exit_status.should == 1 - end - end - - context 'for a debug line' do - it 'calls _on_debug_callback' do - subject.should_receive(:_on_debug_callback).with('1234 a a feh') - subject.line_demuxer('1234 a a feh', 0) - end - end - - context 'for any other line' do - it 'calls _on_other_callback' do - subject.should_receive(:_on_other_callback).with('blah blah blah') - subject.line_demuxer('blah blah blah', 0) - end - end - end - - describe '#format' do - it 'calls the demuxer for each line' do - subject.should_receive(:line_demuxer).exactly(3).times - subject.format("a\nb\nc") - end - - context 'when no lines fail' do - it 'returns 0' do - subject.should_receive(:line_demuxer).with("a\n", 0).ordered.and_return(0) - subject.should_receive(:line_demuxer).with("b\n", 0).ordered.and_return(0) - subject.should_receive(:line_demuxer).with("c", 0).ordered.and_return(0) - subject.format("a\nb\nc").should == 0 - end - end - - context 'when any line fails' do - it 'returns 1' do - subject.should_receive(:line_demuxer).with("a\n", 0).ordered.and_return(0) - subject.should_receive(:line_demuxer).with("b\n", 0).ordered.and_return(1) - subject.should_receive(:line_demuxer).with("c", 1).ordered.and_return(1) - subject.format("a\nb\nc").should == 1 - end - end - end - - describe '#format_build' do - it 'replaces output lines with dots' do - out = capture_stdout do - subject.format_build("a\nb\nc\n") - end - out.should == '...' - end - - it 'ignores empty lines' do - out = capture_stdout do - subject.format_build("\n\n\n") - end - out.should == '' - end - - it 'returns the passed in build text' do - build_text = '' - capture_stdout do - build_text = subject.format_build("a\nb\nc\n") - end - build_text.should == "a\nb\nc\n" - end - - end - - describe '#build_successful build_log' do - it 'displays build successful' do - out = capture_stdout do - subject.build_successful('foo') - end - out.should == "\n\n### Build Successful ###\n\n" - end - - end - - describe '#build_failed build_log, error_log' do - it 'displays the build_log' do - out = capture_stdout do - subject.build_failed('build', 'bar') - end - out.should =~ /build/ - end - - it 'displays the error_log' do - out = capture_stdout do - subject.build_failed('foo', 'error') - end - out.should =~ /error/ - end - - end - end diff --git a/spec/lib/bwoken/formatters/fanout_formatter_spec.rb b/spec/lib/bwoken/formatters/fanout_formatter_spec.rb new file mode 100644 index 0000000..10f370c --- /dev/null +++ b/spec/lib/bwoken/formatters/fanout_formatter_spec.rb @@ -0,0 +1,76 @@ +require 'spec_helper' +require 'bwoken/formatters/fanout_formatter' + +describe Bwoken::FanoutFormatter do + + let(:line_demuxer) { double 'line_demuxer' } + let(:recipients) { [ formatter ] } + let(:formatter) { double 'formatter' } + subject{ described_class.new(line_demuxer) } + before do + subject.add_recipient(formatter) + end + + describe '#format' do + it 'calls the demuxer for each line' do + line_demuxer.should_receive(:demux).exactly(3).times + subject.format("a\nb\nc") + end + + context 'when no lines fail' do + it 'returns 0' do + line_demuxer.should_receive(:demux).with("a\n", 0, recipients).ordered.and_return(0) + line_demuxer.should_receive(:demux).with("b\n", 0, recipients).ordered.and_return(0) + line_demuxer.should_receive(:demux).with("c", 0, recipients).ordered.and_return(0) + subject.format("a\nb\nc").should == 0 + end + end + + context 'when any line fails' do + it 'returns 1' do + line_demuxer.should_receive(:demux).with("a\n", 0, recipients).ordered.and_return(0) + line_demuxer.should_receive(:demux).with("b\n", 0, recipients).ordered.and_return(1) + line_demuxer.should_receive(:demux).with("c", 1, recipients).ordered.and_return(1) + subject.format("a\nb\nc").should == 1 + end + end + end + + describe '#format_build' do + it 'sends output lines to recpients' do + formatter.should_receive(:_on_build_line_callback).with("a\n") + formatter.should_receive(:_on_build_line_callback).with("b\n") + formatter.should_receive(:_on_build_line_callback).with("c\n") + + subject.format_build("a\nb\nc\n") + end + + it 'ignores empty lines' do + formatter.should_receive(:_on_build_line_callback).exactly(0).times + subject.format_build("\n\n\n") + end + + it 'returns the passed in build text' do + formatter.stub(:_on_build_line_callback) + build_text = subject.format_build("a\nb\nc\n") + build_text.should == "a\nb\nc\n" + end + + end + + describe '#build_successful build_log' do + it 'displays build successful' do + formatter.should_receive(:_on_build_successful_callback).with('foo') + subject.build_successful('foo') + end + + end + + describe '#build_failed build_log, error_log' do + it 'displays the build_log' do + formatter.should_receive(:_on_build_failed_callback).with('build', 'bar') + subject.build_failed('build', 'bar') + end + end + +end diff --git a/spec/lib/bwoken/formatters/junit_formatter_spec.rb b/spec/lib/bwoken/formatters/junit_formatter_spec.rb new file mode 100644 index 0000000..9fb2f35 --- /dev/null +++ b/spec/lib/bwoken/formatters/junit_formatter_spec.rb @@ -0,0 +1,162 @@ +require 'spec_helper' +require 'bwoken/formatters/junit_formatter' + +describe Bwoken::JUnitTestSuite do + describe '#initialize' do + it 'sets initial state for an instance' do + expect(subject.test_cases).to be_kind_of Array + expect(subject.test_cases).to have(0).items + expect(subject.tests).to eq(0) + expect(subject.failures).to eq(0) + expect(subject.errors).to eq(0) + end + end + + describe '#complete' do + it 'calculates the correct elapsed time for a test' do + subject.timestamp = Time.now + sleep 0.1 + subject.complete + expect(subject.time.round(1)).to eq(0.1) + end + end + +end + +describe Bwoken::JUnitTestCase do + describe '#initialize' do + it 'sets initial state for an instance' do + expect(subject.logs).to be_kind_of String + expect(subject.error).to be_nil + end + + end + + describe '#complete' do + it 'calculates the correct elapsed time for a test case' do + subject.start_time = Time.now + sleep 0.1 + subject.complete + expect(subject.time.round(1)).to eq(0.1) + end + end +end + +describe Bwoken::JUnitFormatter do + describe '.on' do + it 'increments tests counter when a test is run' do + formatter = Bwoken::JUnitFormatter.new + formatter.test_suites = [Bwoken::JUnitTestSuite.new] + formatter._on_start_callback('2013-10-25 16:10:01 +0000 Start: test one') + expect(formatter.test_suites[0].tests).to eq(1) + end + + it 'increments failure counter when a test fails' do + formatter = Bwoken::JUnitFormatter.new + formatter.test_suites = [Bwoken::JUnitTestSuite.new] + test_case = Bwoken::JUnitTestCase.new + test_case.start_time = Time.now + formatter.test_suites[0].test_cases = [test_case] + formatter._on_fail_callback('2013-10-25 16:10:01 +0000 Fail: login') + expect(formatter.test_suites[0].failures).to eq(1) + end + + it 'increments error counter when a test error occurs' do + formatter = Bwoken::JUnitFormatter.new + formatter.test_suites = [Bwoken::JUnitTestSuite.new] + test_case = Bwoken::JUnitTestCase.new + test_case.start_time = Time.now + formatter.test_suites[0].test_cases = [test_case] + formatter._on_error_callback('2013-10-25 16:10:01 +0000 Error: login') + expect(formatter.test_suites[0].errors).to eq(1) + end + end + + + describe '#generate_report' do + it 'outputs a valid XML report for test suites' do + # Setup + #=================================================================================================================== + now = Time.new(2013, 10, 25, 10, 34, 51, '-05:00') + + test_suite = Bwoken::JUnitTestSuite.new + test_suite.id = 'suite id' + test_suite.package = 'suite package' + test_suite.host_name = 'suite host_name' + test_suite.name = 'suite_name.js' + test_suite.tests = 2 + test_suite.failures = 1 + test_suite.errors = 1 + test_suite.timestamp = now + test_suite.time = 10.0 + + test_case_passed = Bwoken::JUnitTestCase.new + test_case_passed.name = 'test one' + test_case_passed.classname = 'TestOne' + test_case_passed.time = 3.0 + test_case_passed.logs = 'test one logs' + + test_case_failed = Bwoken::JUnitTestCase.new + test_case_failed.name = 'test two' + test_case_failed.classname = 'TestTwo' + test_case_failed.time = 5.0 + test_case_failed.logs = 'test two logs' + test_case_failed.error = 'case 2 error' + + test_suite.test_cases << test_case_passed + test_suite.test_cases << test_case_failed + + subject.test_suites = [test_suite] + + + # Assert + #=================================================================================================================== + subject.stub(:write_results) do |xml, suite_name| + + expect(xml).to be_kind_of(String) + + # Check the test suite + expect(xml.scan(/testsuite\sid="([^"]+)"/)[0]).to include('suite id') + expect(xml.scan(/hostname="([^"]+)"/)[0]).to include('suite host_name') + expect(xml.scan(/testsuite.*name="([^"]+)"/)[0]).to include('suite_name.js') + expect(xml.scan(/testsuite.*tests="([^"]+)"/)[0]).to include('2') + expect(xml.scan(/testsuite.*failures="([^"]+)"/)[0]).to include('1') + expect(xml.scan(/testsuite.*errors="([^"]+)"/)[0]).to include('1') + expect(xml.scan(/testsuite.*time="([^"]+)"/)[0]).to include('10.0') + expect(xml.scan(/testsuite.*timestamp="([^"]+)"/)[0]).to include('2013-10-25 10:34:51 -0500') + + # Check the test cases + expect(xml.scan(/testcase.*\sname="([^"]+)"/)[0]).to include('test one') + expect(xml.scan(/testcase.*\sclassname="([^"]+)"/)[0]).to include('TestOne') + expect(xml.scan(/testcase.*\stime="([^"]+)"/)[0]).to include('3.0') + + expect(xml.scan(/testcase.*\sname="([^"]+)"/)[1]).to include('test two') + expect(xml.scan(/testcase.*\sclassname="([^"]+)"/)[1]).to include('TestTwo') + expect(xml.scan(/testcase.*\stime="([^"]+)"/)[1]).to include('5.0') + + # Check stdout for logs + expect(xml.scan(/system-out.*\n.*\n.*test one logs/)).to have(1).items + expect(xml.scan(/system-err.*\n.*\n.*test two logs/)).to have(1).items + + # Ensure that the resultant document passes XSD validation + xsd = Nokogiri::XML::Schema(File.read(File.expand_path("#{File.dirname(__FILE__)}/../../../support/junit-4.xsd"))) + doc = Nokogiri::XML(xml) + + errors = [] + xsd.validate(doc).each do |error| + puts "Error: #{error}" + errors << error + end + + expect(errors).to have(0).items + + end + + # Test + #=================================================================================================================== + + subject.generate_report + + end + end +end diff --git a/spec/lib/bwoken/formatters/line_demuxer_spec.rb b/spec/lib/bwoken/formatters/line_demuxer_spec.rb new file mode 100644 index 0000000..1282f66 --- /dev/null +++ b/spec/lib/bwoken/formatters/line_demuxer_spec.rb @@ -0,0 +1,61 @@ +require 'spec_helper' +require 'bwoken/formatters/line_demuxer' + +describe Bwoken::LineDemuxer do + + describe '#line_demuxer' do + let(:formatter_one) { double 'format one' } + let(:formatter_two) { double 'format two' } + let(:formatters) { [ formatter_one, formatter_two] } + + context 'for a passing line' do + it 'calls _on_pass_callback and returns correct status' do + expect_call_with(formatters, :_on_pass_callback, '1234 a a Pass') + + exit_status = subject.demux('1234 a a Pass', 0, formatters) + expect(exit_status).to eq(0) + end + end + + context 'for a failing line' do + context 'Fail error' do + it 'calls _on_fail_callback' do + expect_call_with(formatters, :_on_fail_callback, '1234 a a Fail') + exit_status = subject.demux('1234 a a Fail', 0, formatters) + expect(exit_status).to eq(1) + end + end + + context 'Instruments Trace Error message' do + it 'calls _on_fail_callback' do + msg = 'Instruments Trace Error foo' + expect_call_with(formatters, :_on_fail_callback, msg) + exit_status = subject.demux(msg, 0, formatters) + expect(exit_status).to eq(1) + end + end + end + + context 'for a debug line' do + it 'calls _on_debug_callback' do + expect_call_with(formatters, :_on_debug_callback, '1234 a a feh') + exit_status = subject.demux('1234 a a feh', 0, formatters) + expect(exit_status).to eq(0) + end + end + + context 'for any other line' do + it 'calls _on_other_callback' do + expect_call_with(formatters, :_on_other_callback, 'blah blah blah') + exit_status = subject.demux('blah blah blah', 0, formatters) + expect(exit_status).to eq(0) + end + end + end + + def expect_call_with(formatters, method, args) + formatters.each do |f| + f.should_receive(method).with(args) + end + end +end diff --git a/spec/lib/bwoken/script_spec.rb b/spec/lib/bwoken/script_spec.rb index eb0ea52..5c1f14f 100644 --- a/spec/lib/bwoken/script_spec.rb +++ b/spec/lib/bwoken/script_spec.rb @@ -17,11 +17,13 @@ class Simulator; end before do subject.formatter.stub(:format).and_return(exit_status) subject.formatter.stub(:before_script_run) + subject.formatter.stub(:after_script_run) end it 'outputs that a script is about to run' do subject.path = 'path' subject.formatter.should_receive(:before_script_run).with('path') + subject.formatter.should_receive(:after_script_run) Open3.stub(:popen3) subject.stub(:cmd) subject.run diff --git a/spec/support/junit-4.xsd b/spec/support/junit-4.xsd new file mode 100644 index 0000000..a2da311 --- /dev/null +++ b/spec/support/junit-4.xsd @@ -0,0 +1,91 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +