Only embed declared public headers in the framework

As pointed out by @neonichu.

lib/cocoapods/installer/target_installer/pod_target_installer.rb

@@ -49,6 +49,7 @@ module Pod
           consumer = file_accessor.spec_consumer
 
           headers = file_accessor.headers
+          public_headers = file_accessor.public_headers
           other_source_files = file_accessor.source_files.select { |sf| sf.extname == '.d' }
 
           {
@@ -65,10 +66,12 @@ module Pod
           native_target.add_file_references(header_file_refs) do |build_file|
             # Set added headers as public if needed
             if target.requires_framework?
+              if public_headers.include?(build_file.file_ref.real_path)
                 build_file.settings ||= {}
                 build_file.settings['ATTRIBUTES'] = ['Public']
               end
             end
+          end
 
           other_file_refs = other_source_files.map { |sf| project.reference_for_path(sf) }
           native_target.add_file_references(other_file_refs, nil)