tweaks to cert grabbing and metadata parsing

socmap · Feb 18, 2017 · c3429be · c3429be
1 parent 66d3ad5
commit c3429be
Show file tree

Hide file tree

Showing 2 changed files with 17 additions and 8 deletions.
diff --git a/lib/tasks/helpers/parse.rb b/lib/tasks/helpers/parse.rb
@@ -195,17 +195,20 @@ def download_and_extract_metadata(uri,extract_content=true)
       if yomu.metadata["Content-Type"] == "application/pdf"
 
         _create_entity "Person",
-          { "name" => yomu.metadata["Author"], "uri" => uri } if yomu.metadata["Author"]
+          { "name" => yomu.metadata["Author"], "source_uri" => uri } if yomu.metadata["Author"]
         _create_entity "SoftwarePackage",
-        { "name" => "#{yomu.metadata["xmp:CreatorTool"]}", "plugin" => "#{yomu.metadata["producer"]}", "uri" => uri } if yomu.metadata["producer"]
+        { "name" => "#{yomu.metadata["xmp:CreatorTool"]} / #{yomu.metadata["producer"] }",
+          "creator" => "#{yomu.metadata["xmp:CreatorTool"]}",
+          "producer" => "#{yomu.metadata["producer"]}",
+          "source_uri" => uri } if (yomu.metadata["producer"] || yomu.metadata["xmp:CreatorTool"])
 
       # Handle MP3/4
       elsif yomu.metadata["Content-Type"] == "audio/mpeg"
-        _create_entity "Person", {"name" => yomu.metadata["meta:author"], "uri" => uri }
-        _create_entity "Person", {"name" => yomu.metadata["creator"], "uri" => uri }
-        _create_entity "Person", {"name" => yomu.metadata["xmpDM:artist"] }
+        _create_entity "Person", {"name" => yomu.metadata["meta:author"], "source_uri" => uri }
+        _create_entity "Person", {"name" => yomu.metadata["creator"], "source_uri" => uri }
+        _create_entity "Person", {"name" => yomu.metadata["xmpDM:artist"], "source_uri" => uri }
       else
-        _create_entity "Info", {"name" => "Metadata (#{uri})", "content" => yomu.metadata.to_json, "uri" => "#{uri}" }
+        _create_entity "Info", {"name" => "Metadata (#{uri})", "content" => yomu.metadata.to_json, "uri" => "#{uri}", "source_uri" => uri }
       end
 
       # Look for entities in the text of the entity
@@ -217,7 +220,7 @@ def download_and_extract_metadata(uri,extract_content=true)
     rescue JSON::ParserError => e
       @task_result.logger.log "ERROR parsing JSON: #{e}"
     rescue Errno::EPIPE => e
-      @task_result.logger.log "ERROR Unable to contact Tika server: #{e}"
+      @task_result.logger.log "ERROR Unable to contact Tika: #{e}"
     rescue OpenURI::HTTPError => e     # don't die if we can't find the file
       @task_result.logger.log "ERROR Unable to download file: #{e}"
     rescue URI::InvalidURIError => e     # handle invalid uris

diff --git a/lib/tasks/uri_gather_ssl_certificate.rb b/lib/tasks/uri_gather_ssl_certificate.rb
@@ -19,7 +19,8 @@ def self.metadata
       :allowed_options => [
         {:name => "skip_incapsula", :type => "Boolean", :regex => "boolean", :default => true },
         {:name => "skip_cloudflare", :type => "Boolean", :regex => "boolean", :default => true },
-        {:name => "skip_distil", :type => "Boolean", :regex => "boolean", :default => true }
+        {:name => "skip_distil", :type => "Boolean", :regex => "boolean", :default => true },
+        {:name => "skip_fastly", :type => "Boolean", :regex => "boolean", :default => true }
       ],
       :created_types => ["DnsRecord","SslCertificate"]
     }
@@ -31,6 +32,7 @@ def run
     opt_skip_cloudflare = _get_option "skip_cloudflare"
     opt_skip_distill = _get_option "skip_distill"
     opt_skip_incapsula = _get_option "skip_incapsula"
+    opt_skip_fastly = _get_option "skip_fastly"
 
     uri = _get_entity_name
 
@@ -73,6 +75,10 @@ def run
               return
             end
 
+            if alt_name =~ /fastly.net$/ && opt_skip_fastly
+              _log "This is a fastly certificate, skipping further entity creation"
+              return
+            end
 
 
             # Remove any leading wildcards so we get a sensible domain name