Add validator support to run test specs during lint

CHANGELOG.md

@@ -8,6 +8,10 @@ To install release candidates run `[sudo] gem install cocoapods --pre`
 
 ##### Enhancements
 
+* Add validator support to run test specs during lint  
+  [Dimitris Koutsogiorgas](https://github.com/dnkoutso)
+  [#6753](https://github.com/CocoaPods/CocoaPods/pull/6753)
+
 * Fix to include proper runtime search paths for test native targets  
   [Dimitris Koutsogiorgas](https://github.com/dnkoutso)
   [#6727](https://github.com/CocoaPods/CocoaPods/pull/6727)

lib/cocoapods/command/lib/lint.rb

@@ -24,6 +24,7 @@ module Pod
             ['--swift-version=VERSION', 'The SWIFT_VERSION that should be used to lint the spec. ' \
              'This takes precedence over a .swift-version file.'],
             ['--skip-import-validation', 'Lint skips validating that the pod can be imported'],
+            ['--skip-tests', 'Lint skips building and running tests during validation'],
           ].concat(super)
         end
 
@@ -39,6 +40,7 @@ module Pod
           @private         = argv.flag?('private', false)
           @swift_version   = argv.option('swift-version', nil)
           @skip_import_validation = argv.flag?('skip-import-validation', false)
+          @skip_tests = argv.flag?('skip-tests', false)
           @podspecs_paths = argv.arguments!
           super
         end
@@ -62,6 +64,7 @@ module Pod
             validator.ignore_public_only_results = @private
             validator.swift_version = @swift_version
             validator.skip_import_validation = @skip_import_validation
+            validator.skip_tests = @skip_tests
             validator.validate
 
             unless @clean

lib/cocoapods/command/spec/lint.rb

@@ -30,6 +30,7 @@ module Pod
             ['--swift-version=VERSION', 'The SWIFT_VERSION that should be used to lint the spec. ' \
              'This takes precedence over a .swift-version file.'],
             ['--skip-import-validation', 'Lint skips validating that the pod can be imported'],
+            ['--skip-tests', 'Lint skips building and running tests during validation'],
           ].concat(super)
         end
 
@@ -45,6 +46,7 @@ module Pod
           @private         = argv.flag?('private', false)
           @swift_version   = argv.option('swift-version', nil)
           @skip_import_validation = argv.flag?('skip-import-validation', false)
+          @skip_tests = argv.flag?('skip-tests', false)
           @podspecs_paths = argv.arguments!
           super
         end
@@ -64,6 +66,7 @@ module Pod
             validator.ignore_public_only_results = @private
             validator.swift_version = @swift_version
             validator.skip_import_validation = @skip_import_validation
+            validator.skip_tests = @skip_tests
             validator.validate
             failure_reasons << validator.failure_reason
 

lib/cocoapods/installer/xcode/pods_project_generator/pod_target_installer.rb

@@ -134,7 +134,7 @@ module Pod
             target.file_accessors.each do |file_accessor|
               consumer = file_accessor.spec_consumer
 
-              native_target = native_target_for_consumer(consumer)
+              native_target = target.native_target_for_spec(consumer.spec)
               headers = file_accessor.headers
               public_headers = file_accessor.public_headers
               private_headers = file_accessor.private_headers
@@ -167,40 +167,6 @@ module Pod
             end
           end
 
-          # Returns the corresponding native target to use based on the provided consumer.
-          # This is used to figure out whether to add a source file into the library native target or any of the
-          # test native targets.
-          #
-          # @param  [Consumer] consumer
-          #         The consumer to base from in order to find the native target.
-          #
-          # @return [PBXNativeTarget] the native target to use or `nil` if none is found.
-          #
-          def native_target_for_consumer(consumer)
-            return native_target unless consumer.spec.test_specification?
-            target.test_native_targets.find do |native_target|
-              native_target.symbol_type == product_type_for_test_type(consumer.spec.test_type)
-            end
-          end
-
-          # Returns the corresponding native product type to use given the test type.
-          # This is primarily used when creating the native targets in order to produce the correct test bundle target
-          # based on the type of tests included.
-          #
-          # @param  [Symbol] test_type
-          #         The test type to map to provided by the test specification DSL.
-          #
-          # @return [Symbol] The native product type to use.
-          #
-          def product_type_for_test_type(test_type)
-            case test_type
-            when :unit
-              :unit_test_bundle
-            else
-              raise Informative, "Unknown test type passed `#{test_type}`."
-            end
-          end
-
           # Adds the test targets for the library to the Pods project with the
           # appropriate build configurations.
           #
@@ -208,7 +174,7 @@ module Pod
           #
           def add_test_targets
             target.supported_test_types.each do |test_type|
-              product_type = product_type_for_test_type(test_type)
+              product_type = target.product_type_for_test_type(test_type)
               name = target.test_target_label(test_type)
               platform = target.platform.name
               language = target.uses_swift? ? :swift : :objc
@@ -257,7 +223,7 @@ module Pod
                   bundle_target.add_resources(resource_phase_refs + compile_phase_refs)
                 end
 
-                native_target = native_target_for_consumer(file_accessor.spec_consumer)
+                native_target = target.native_target_for_spec(file_accessor.spec_consumer.spec)
                 target.user_build_configurations.each do |bc_name, type|
                   bundle_target.add_build_configuration(bc_name, type)
                 end

lib/cocoapods/target/pod_target.rb

@@ -192,6 +192,40 @@ module Pod
       specs.select(&:test_specification?).map(&:test_type).uniq
     end
 
+    # Returns the corresponding native target to use based on the provided specification.
+    # This is used to figure out whether to add a source file into the library native target or any of the
+    # test native targets.
+    #
+    # @param  [Specification] spec
+    #         The specifcation to base from in order to find the native target.
+    #
+    # @return [PBXNativeTarget] the native target to use or `nil` if none is found.
+    #
+    def native_target_for_spec(spec)
+      return native_target unless spec.test_specification?
+      test_native_targets.find do |native_target|
+        native_target.symbol_type == product_type_for_test_type(spec.test_type)
+      end
+    end
+
+    # Returns the corresponding native product type to use given the test type.
+    # This is primarily used when creating the native targets in order to produce the correct test bundle target
+    # based on the type of tests included.
+    #
+    # @param  [Symbol] test_type
+    #         The test type to map to provided by the test specification DSL.
+    #
+    # @return [Symbol] The native product type to use.
+    #
+    def product_type_for_test_type(test_type)
+      case test_type
+      when :unit
+        :unit_test_bundle
+      else
+        raise Informative, "Unknown test type `#{test_type}`."
+      end
+    end
+
     # @return [Specification] The root specification for the target.
     #
     def root_spec

lib/cocoapods/validator.rb

@@ -72,7 +72,7 @@ module Pod
       a_spec = spec
       if spec && @only_subspec
         subspec_name = @only_subspec.start_with?(spec.root.name) ? @only_subspec : "#{spec.root.name}/#{@only_subspec}"
-        a_spec = spec.subspec_by_name(subspec_name)
+        a_spec = spec.subspec_by_name(subspec_name, true, true)
         @subspec_name = a_spec.name
       end
 
@@ -186,10 +186,14 @@ module Pod
     #
     attr_accessor :only_subspec
 
-    # @return [Bool] Whether the validator should validate all subspecs
+    # @return [Bool] Whether the validator should validate all subspecs.
     #
     attr_accessor :no_subspecs
 
+    # @return [Bool] Whether the validator should skip building and running tests.
+    #
+    attr_accessor :skip_tests
+
     # @return [Bool] Whether frameworks should be used for the installation.
     #
     attr_accessor :use_frameworks
@@ -287,6 +291,10 @@ module Pod
     # Perform analysis for a given spec (or subspec)
     #
     def perform_extensive_analysis(spec)
+      if spec.test_specification?
+        error('spec', "Validating a test spec (`#{spec.name}`) is not supported.")
+        return false
+      end
       validate_homepage(spec)
       validate_screenshots(spec)
       validate_social_media_url(spec)
@@ -305,6 +313,7 @@ module Pod
           add_app_project_import
           validate_vendored_dynamic_frameworks
           build_pod
+          test_pod unless skip_tests
         ensure
           tear_down_validation_environment
         end
@@ -322,7 +331,7 @@ module Pod
     # Recursively perform the extensive analysis on all subspecs
     #
     def perform_extensive_subspec_analysis(spec)
-      spec.subspecs.send(fail_fast ? :all? : :each) do |subspec|
+      spec.subspecs.reject(&:test_specification?).send(fail_fast ? :all? : :each) do |subspec|
         @subspec_name = subspec.name
         perform_extensive_analysis(subspec)
       end
@@ -410,7 +419,7 @@ module Pod
     end
 
     def download_pod
-      podfile = podfile_from_spec(consumer.platform_name, deployment_target, use_frameworks)
+      podfile = podfile_from_spec(consumer.platform_name, deployment_target, use_frameworks, consumer.spec.test_specs.map(&:name))
       sandbox = Sandbox.new(config.sandbox_root)
       @installer = Installer.new(sandbox, podfile)
       @installer.use_default_plugins = false
@@ -496,7 +505,7 @@ module Pod
       deployment_target = spec.subspec_by_name(subspec_name).deployment_target(consumer.platform_name)
       @installer.aggregate_targets.each do |target|
         target.pod_targets.each do |pod_target|
-          next unless native_target = pod_target.native_target
+          next unless (native_target = pod_target.native_target)
           native_target.build_configuration_list.build_configurations.each do |build_configuration|
             (build_configuration.build_settings['OTHER_CFLAGS'] ||= '$(inherited)') << ' -Wincomplete-umbrella'
             build_configuration.build_settings['SWIFT_VERSION'] = swift_version if pod_target.uses_swift?
@@ -537,28 +546,38 @@ module Pod
         UI.warn "Skipping compilation with `xcodebuild' because it can't be found.\n".yellow
       else
         UI.message "\nBuilding with xcodebuild.\n".yellow do
-          output = xcodebuild
+          scheme = if skip_import_validation?
+                     @installer.pod_targets.find { |pt| pt.pod_name == spec.root.name }.label
+                   else
+                     'App'
+                   end
+          output = xcodebuild('build', scheme, 'Release')
           UI.puts output
           parsed_output = parse_xcodebuild_output(output)
-          parsed_output.each do |message|
-            # Checking the error for `InputFile` is to work around an Xcode
-            # issue where linting would fail even though `xcodebuild` actually
-            # succeeds. Xcode.app also doesn't fail when this issue occurs, so
-            # it's safe for us to do the same.
-            #
-            # For more details see https://github.com/CocoaPods/CocoaPods/issues/2394#issuecomment-56658587
-            #
-            if message.include?("'InputFile' should have")
-              next
-            end
+          translate_output_to_linter_messages(parsed_output)
+        end
+      end
+    end
 
-            if message =~ /\S+:\d+:\d+: error:/
-              error('xcodebuild', message)
-            elsif message =~ /\S+:\d+:\d+: warning:/
-              warning('xcodebuild', message)
-            else
-              note('xcodebuild', message)
-            end
+    # Builds and runs all test sources associated with the current specification being validated.
+    #
+    # @note   Xcode warnings are treaded as notes because the spec maintainer
+    #         might not be the author of the library
+    #
+    # @return [void]
+    #
+    def test_pod
+      if !xcodebuild_available?
+        UI.warn "Skipping test validation with `xcodebuild' because it can't be found.\n".yellow
+      else
+        UI.message "\nTesting with xcodebuild.\n".yellow do
+          pod_target = @installer.pod_targets.find { |pt| pt.pod_name == spec.root.name }
+          consumer.spec.test_specs.each do |test_spec|
+            scheme = pod_target.native_target_for_spec(test_spec)
+            output = xcodebuild('test', scheme, 'Debug')
+            UI.puts output
+            parsed_output = parse_xcodebuild_output(output)
+            translate_output_to_linter_messages(parsed_output)
           end
         end
       end
@@ -678,6 +697,29 @@ module Pod
       add_result(:note, *args)
     end
 
+    def translate_output_to_linter_messages(parsed_output)
+      parsed_output.each do |message|
+        # Checking the error for `InputFile` is to work around an Xcode
+        # issue where linting would fail even though `xcodebuild` actually
+        # succeeds. Xcode.app also doesn't fail when this issue occurs, so
+        # it's safe for us to do the same.
+        #
+        # For more details see https://github.com/CocoaPods/CocoaPods/issues/2394#issuecomment-56658587
+        #
+        if message.include?("'InputFile' should have")
+          next
+        end
+
+        if message =~ /\S+:\d+:\d+: error:/
+          error('xcodebuild', message)
+        elsif message =~ /\S+:\d+:\d+: warning:/
+          warning('xcodebuild', message)
+        else
+          note('xcodebuild', message)
+        end
+      end
+    end
+
     def shares_pod_target_xcscheme?(pod_target)
       Pathname.new(@installer.pods_project.path + pod_target.label).exist?
     end
@@ -727,13 +769,16 @@ module Pod
     # @param  [Bool] use_frameworks
     #         whether frameworks should be used for the installation
     #
+    # @param [Array<String>] test_spec_names
+    #         the test spec names to include in the podfile.
+    #
     # @return [Podfile] a podfile that requires the specification on the
     #         current platform.
     #
     # @note   The generated podfile takes into account whether the linter is
     #         in local mode.
     #
-    def podfile_from_spec(platform_name, deployment_target, use_frameworks = true)
+    def podfile_from_spec(platform_name, deployment_target, use_frameworks = true, test_spec_names = [])
       name     = subspec_name || spec.name
       podspec  = file.realpath
       local    = local?
@@ -749,6 +794,13 @@ module Pod
           else
             pod name, :podspec => podspec.to_s
           end
+          test_spec_names.each do |test_spec_name|
+            if local
+              pod test_spec_name, :path => podspec.dirname.to_s
+            else
+              pod test_spec_name, :podspec => podspec.to_s
+            end
+          end
         end
       end
     end
@@ -779,14 +831,9 @@ module Pod
     # @return [String] Executes xcodebuild in the current working directory and
     #         returns its output (both STDOUT and STDERR).
     #
-    def xcodebuild
+    def xcodebuild(action, scheme, configuration)
       require 'fourflusher'
-      scheme = if skip_import_validation?
-                 @installer.pod_targets.find { |pt| pt.pod_name == spec.root.name }.label
-               else
-                 'App'
-      end
-      command = %W(clean build -workspace #{File.join(validation_dir, 'App.xcworkspace')} -scheme #{scheme} -configuration Release)
+      command = %W(clean #{action} -workspace #{File.join(validation_dir, 'App.xcworkspace')} -scheme #{scheme} -configuration #{configuration})
       case consumer.platform_name
       when :osx, :macos
         command += %w(CODE_SIGN_IDENTITY=)

cocoapods-integration-specs @ a2933449

-Subproject commit 0712f81b44cec8cd0fd4add05c829afe49ebf0e8
+Subproject commit a2933449730c7c103312369759fb21dbd825ec78

spec/functional/command/repo/push_spec.rb

@@ -171,10 +171,10 @@ module Pod
     end
 
     it 'validates specs as frameworks by default' do
-      Validator.any_instance.expects(:podfile_from_spec).with(:ios, '8.0', true).times(3)
-      Validator.any_instance.expects(:podfile_from_spec).with(:osx, nil, true).twice
-      Validator.any_instance.expects(:podfile_from_spec).with(:watchos, nil, true).twice
-      Validator.any_instance.expects(:podfile_from_spec).with(:tvos, nil, true).twice
+      Validator.any_instance.expects(:podfile_from_spec).with(:ios, '8.0', true, []).times(3)
+      Validator.any_instance.expects(:podfile_from_spec).with(:osx, nil, true, []).twice
+      Validator.any_instance.expects(:podfile_from_spec).with(:watchos, nil, true, []).twice
+      Validator.any_instance.expects(:podfile_from_spec).with(:tvos, nil, true, []).twice
 
       cmd = command('repo', 'push', 'master')
       # Git push will throw an exception here since this is a local custom git repo. All we care is the validator
@@ -185,10 +185,10 @@ module Pod
     end
 
     it 'validates specs as libraries if requested' do
-      Validator.any_instance.expects(:podfile_from_spec).with(:ios, nil, false).times(3)
-      Validator.any_instance.expects(:podfile_from_spec).with(:osx, nil, false).twice
-      Validator.any_instance.expects(:podfile_from_spec).with(:watchos, nil, false).twice
-      Validator.any_instance.expects(:podfile_from_spec).with(:tvos, nil, false).twice
+      Validator.any_instance.expects(:podfile_from_spec).with(:ios, nil, false, []).times(3)
+      Validator.any_instance.expects(:podfile_from_spec).with(:osx, nil, false, []).twice
+      Validator.any_instance.expects(:podfile_from_spec).with(:watchos, nil, false, []).twice
+      Validator.any_instance.expects(:podfile_from_spec).with(:tvos, nil, false, []).twice
 
       cmd = command('repo', 'push', 'master', '--use-libraries')
       # Git push will throw an exception here since this is a local custom git repo. All we care is the validator

spec/unit/installer/xcode/pods_project_generator/pod_target_installer_spec.rb

@@ -221,25 +221,6 @@ module Pod
                 content = @coconut_pod_target.umbrella_header_path.read
                 content.should.not =~ /"CoconutTestHeader.h"/
               end
-
-              it 'returns the correct native target based on the consumer provided' do
-                @installer.install!
-                native_target = @installer.send(:native_target_for_consumer, @coconut_spec.consumer(:ios))
-                native_target.name.should == 'CoconutLib'
-                native_target.product_reference.name.should == 'libCoconutLib.a'
-                test_native_target = @installer.send(:native_target_for_consumer, @coconut_spec.test_specs.first.consumer(:ios))
-                test_native_target.name.should == 'CoconutLib-Unit-Tests'
-                test_native_target.product_reference.name.should == 'CoconutLib-Unit-Tests'
-              end
-
-              it 'returns the correct product type for test type' do
-                @installer.send(:product_type_for_test_type, :unit).should == :unit_test_bundle
-              end
-
-              it 'raises for unknown test type' do
-                exception = lambda { @installer.send(:product_type_for_test_type, :weird_test_type) }.should.raise Informative
-                exception.message.should.include 'Unknown test type passed `weird_test_type`.'
-              end
             end
 
             #--------------------------------------#

spec/unit/target/pod_target_spec.rb

@@ -311,10 +311,10 @@ module Pod
 
       describe 'test spec support' do
         before do
-          spec = fixture_spec('coconut-lib/CoconutLib.podspec')
+          @coconut_spec = fixture_spec('coconut-lib/CoconutLib.podspec')
           @test_spec_target_definition = Podfile::TargetDefinition.new('Pods', nil)
           @test_spec_target_definition.abstract = false
-          @test_pod_target = PodTarget.new([spec, *spec.recursive_subspecs], [@test_spec_target_definition], config.sandbox)
+          @test_pod_target = PodTarget.new([@coconut_spec, *@coconut_spec.recursive_subspecs], [@test_spec_target_definition], config.sandbox)
           @test_pod_target.stubs(:platform).returns(:ios)
         end
 
@@ -329,6 +329,26 @@ module Pod
         it 'returns test label based on test type' do
           @test_pod_target.test_target_label(:unit).should == 'CoconutLib-Unit-Tests'
         end
+
+        it 'returns the correct native target based on the consumer provided' do
+          @test_pod_target.stubs(:native_target).returns(stub(:name => 'CoconutLib', :symbol_type => :dynamic_library, :product_reference => stub(:name => 'libCoconutLib.a')))
+          @test_pod_target.stubs(:test_native_targets).returns([stub(:name => 'CoconutLib-Unit-Tests', :symbol_type => :unit_test_bundle, :product_reference => stub(:name => 'CoconutLib-Unit-Tests'))])
+          native_target = @test_pod_target.native_target_for_spec(@coconut_spec)
+          native_target.name.should == 'CoconutLib'
+          native_target.product_reference.name.should == 'libCoconutLib.a'
+          test_native_target = @test_pod_target.native_target_for_spec(@coconut_spec.test_specs.first)
+          test_native_target.name.should == 'CoconutLib-Unit-Tests'
+          test_native_target.product_reference.name.should == 'CoconutLib-Unit-Tests'
+        end
+
+        it 'returns the correct product type for test type' do
+          @test_pod_target.product_type_for_test_type(:unit).should == :unit_test_bundle
+        end
+
+        it 'raises for unknown test type' do
+          exception = lambda { @test_pod_target.product_type_for_test_type(:weird_test_type) }.should.raise Informative
+          exception.message.should.include 'Unknown test type `weird_test_type`.'
+        end
       end
     end
   end

spec/unit/validator_spec.rb

@@ -335,11 +335,11 @@ module Pod
           s.ios.deployment_target = '7.0'
         end
         validator.spec.stubs(:subspecs).returns([subspec])
-        validator.expects(:podfile_from_spec).with(:osx, nil, nil).once
-        validator.expects(:podfile_from_spec).with(:ios, nil, nil).once
-        validator.expects(:podfile_from_spec).with(:ios, '7.0', nil).once
-        validator.expects(:podfile_from_spec).with(:tvos, nil, nil).once
-        validator.expects(:podfile_from_spec).with(:watchos, nil, nil).once
+        validator.expects(:podfile_from_spec).with(:osx, nil, nil, []).once
+        validator.expects(:podfile_from_spec).with(:ios, nil, nil, []).once
+        validator.expects(:podfile_from_spec).with(:ios, '7.0', nil, []).once
+        validator.expects(:podfile_from_spec).with(:tvos, nil, nil, []).once
+        validator.expects(:podfile_from_spec).with(:watchos, nil, nil, []).once
         validator.send(:perform_extensive_analysis, validator.spec)
       end
 
@@ -819,10 +819,10 @@ module Pod
 
         setup_validator
 
-        @validator.expects(:podfile_from_spec).with(:osx, nil, true).once
-        @validator.expects(:podfile_from_spec).with(:ios, '8.0', true).once
-        @validator.expects(:podfile_from_spec).with(:tvos, nil, true).once
-        @validator.expects(:podfile_from_spec).with(:watchos, nil, true).once
+        @validator.expects(:podfile_from_spec).with(:osx, nil, true, []).once
+        @validator.expects(:podfile_from_spec).with(:ios, '8.0', true, []).once
+        @validator.expects(:podfile_from_spec).with(:tvos, nil, true, []).once
+        @validator.expects(:podfile_from_spec).with(:watchos, nil, true, []).once
         @validator.send(:perform_extensive_analysis, @validator.spec)
       end
 
@@ -831,12 +831,20 @@ module Pod
 
         setup_validator
 
-        @validator.expects(:podfile_from_spec).with(:osx, nil, false).once
-        @validator.expects(:podfile_from_spec).with(:ios, nil, false).once
-        @validator.expects(:podfile_from_spec).with(:tvos, nil, false).once
-        @validator.expects(:podfile_from_spec).with(:watchos, nil, false).once
+        @validator.expects(:podfile_from_spec).with(:osx, nil, false, []).once
+        @validator.expects(:podfile_from_spec).with(:ios, nil, false, []).once
+        @validator.expects(:podfile_from_spec).with(:tvos, nil, false, []).once
+        @validator.expects(:podfile_from_spec).with(:watchos, nil, false, []).once
         @validator.send(:perform_extensive_analysis, @validator.spec)
       end
+
+      it 'shows an error when performing extensive analysis on a test spec' do
+        setup_validator
+        subspec = Specification.new(@validator.spec, 'Tests', true)
+        @validator.send(:perform_extensive_analysis, subspec)
+        @validator.results.map(&:to_s).first.should.include 'Validating a test spec (`JSONKit/Tests`) is not supported.'
+        @validator.result_type.should == :error
+      end
     end
 
     describe 'dynamic binaries validation' do