Finished up conversion for literacy standards

chisel/lib/chisel.rb

@@ -18,11 +18,50 @@
 identifiers_v2 = CCSSIIdentifier.new('raw-data/CCSSI/CCSSI-identifiers-v2.json')
 CCSSIIdentifier.combine_v1_with_v2(identifiers_v1.doc, identifiers_v2.doc)
 
-asn_reading.clean.
-            flatten.
-            munge
+asn_reading.clean
+           .flatten
+           .nest
+           .munge
+           .assign_guids(identifiers_v2.doc)
+           .organize
 
-puts MultiJson.encode(asn_reading.doc, :pretty => true)
-#puts MultiJson.encode(identifiers_v1.doc, :pretty => true)
-#puts MultiJson.encode(identifiers_v2.doc, :pretty => true)
 
+#################################
+#
+# Check for missing standards
+#
+##################################
+
+# Litaracy
+
+asn_ids = asn_reading.doc.collect{|s| s['id']}
+asn_ids.uniq!
+
+literacy_ids = identifiers_v2.literacy.collect{|s| s['GUID']}
+literacy_ids.uniq!
+diff = literacy_ids - asn_ids
+#missing_standards = identifiers_v2.literacy.collect do |id|
+    #next unless diff.find{|s| s == id['GUID']}
+    #id
+#end.compact!
+puts diff.count
+
+# Math
+
+
+#math_ids = identifiers_v2.math.collect{|s| s['GUID']}
+#math_ids.uniq!
+#diff = math_ids - asn_ids
+
+#puts diff.count
+
+#####################################
+#
+# Write file
+#
+# ##################################
+#
+
+f = File.open('clean-data/CC/literacy/literacy-0.8.0.json', 'w+')
+f.write(MultiJson.encode(asn_reading.doc, :pretty => true))
+f.close

chisel/lib/chisel/asn_flattener.rb

@@ -3,23 +3,24 @@ module ASNFlattener
 
   def flatten
     flat = []
-    self.doc.each do |branch|
-      flat << branch
-      push_children(flat, branch)
+    self.doc.each_with_index do |doc, i|
+      flat << doc
+      push_children(flat, doc)
     end
     self.doc = flat
     self
   end
 
   def push_children(container, element)
     if element.is_a?(Hash) && element.has_key?("children")
+      ids = element['children'].collect{|el| el['id']}
       element['children'].each do |el|
-        el[:asnParent] = element['id']
+        el['parent'] = element['id']
         container << el 
           push_children(container, el)
         end
       end
-      element.delete("children")
+      element["children"] = ids
   end
 
 end

chisel/lib/chisel/asn_manifest.rb

@@ -1,11 +1,13 @@
 require_relative 'asn_cleaner'
 require_relative 'asn_flattener'
 require_relative 'asn_munger'
+require_relative 'asn_organizer'
 
 class ASNManifest
   include ASNCleaner
   include ASNFlattener
   include ASNMunger
+  include ASNOrganizer
 
   def initialize(file)
     @file = File.open(file, 'r+').read
@@ -24,5 +26,27 @@ def doc
   end
 
 
+  def find_guid(code, guids)
+    guids.find do |guid|
+      guid["short_code_v1"] == code
+    end
+  end
+
+  def assign_guids(guids)
+    doc.each_with_index do |standard, i|
+      guid = find_guid(standard["CCSS"]["simplifiedDotNotation"], guids)
+      if guid.nil?
+        #puts standard["id"]
+      else
+        doc[i]["id"]                    = guid["GUID"]
+        doc[i]["CCSS"]["URI"]           = guid["URI"]
+        doc[i]["CCSS"]["dotNotationV1"] = guid["dot_notation_v1"]
+        doc[i]["CCSS"]["dotNotation"]   = guid["dot_notation"]
+        doc[i]["CCSS"]["currentURL"]    = guid["current_url"]
+      end
+    end
+    self
+  end
+
 
 end

chisel/lib/chisel/asn_munger.rb

@@ -2,57 +2,35 @@ module ASNMunger
 
   def munge
     doc.each_with_index do |standard, i|
-      move_pref_label(doc[i])
-      prefix_asn_fields(doc[i])
+      add_ccss_notations(doc[i])
       clean_comments(doc[i])
-      remove_number_from_standard(doc[i])
+      remove_number_from_statement(doc[i])
       flatten_education_level(doc[i])
       rename_education_level(doc[i])
       change_statement_notation(doc[i])
       add_jurisdiction(doc[i])
       convert_grade_levels_to_ceds(doc[i])
+      standards_without_codes(doc[i])
+      add_grade_level(doc[i])
     end
+    self
   end
 
-  def move_pref_label(standard)
-    standard.each do |k, v|
-      next unless v.is_a?(Hash) 
-      # move prefLabel 
-      # eg. move from 
-      #
-      # "authorityStatus": {
-      #  "uri": "Original",
-      #  "prefLabel": "Original Statement"
-      # },
-      #
-      # to
-      # "authorityStatus": "Original Statement"
-      if v.has_key?("prefLabel")
-        standard[k] = v["prefLabel"]
-      end
 
-      # move literal to value
-      # from: 
-      # "description": {
-      #  "literal": "Standards for Mathematical Practice",
-      #  "language": "en-US"
-      # },
-      # 
-      # to:
-      # "description" : "Standards for Mathematical Practice"
-      #
-      if v.has_key?("literal")
-        standard[k] = v["literal"]
-      end
+  def add_ccss_notations(standard)
+    file = File.open('raw-data/CCSSI/asn_to_ccss_notation.json', 'r').read
+    doc = MultiJson.decode(file)
+    found = doc.find do |d|
+      d['asn_notation'] == standard['ASN']['statementNotation']
     end
+    return if found.nil?
+    found['ccss_notation'].gsub!(/CCSS.ELA-Literacy./, '')
+    found['ccss_notation'].gsub!(/CCSS.Math.Content./, '')
+    found['ccss_notation'].gsub!(/CCSS.Math.Practice./, '')
+    standard['CCSS']['simplifiedDotNotation'] = found['ccss_notation']
   end
 
-  def prefix_asn_fields(standard)
-    standard["asnIdentifier"] = standard.delete("identifier")
-    standard["asnStatementNotation"] = standard.delete("statementNotation") if standard.has_key?("statementNotation")
-    standard["asnStatementNotation"] = standard.delete("statementNotation") if standard.has_key?("statementNotation")
-
-  end
+
 
   def clean_comments(standard)
     if standard.has_key?("comment") && standard.class == Array
@@ -62,7 +40,7 @@ def clean_comments(standard)
 
 
   # parse out number from standard
-  def remove_number_from_standard(standard)
+  def remove_number_from_statement(standard)
     return unless standard.has_key?("description")
     num = ""
     text = ""
@@ -86,14 +64,17 @@ def rename_education_level(standard)
     standard["gradeLevels"] = Array(standard.delete("educationLevel")) if standard.has_key?("educationLevel")
   end 
 
-  #change statement notation to match NGA/CCSSO notation
+  # change statement notation to match NGA/CCSSO notation
+  # e.g.: WHST.9-10.2.b => WHST.9-10.2b
   def change_statement_notation(standard)
-    return unless standard.has_key?("asnStatementNotation")
-    code = standard['asnStatementNotation'].clone
+    return if standard['ASN']['statementNotation'].nil?
+    return unless standard['CCSS']['simplifiedDotNotation'].nil?
+
+    code = standard["ASN"]['statementNotation'].clone
     if code.gsub(/(\d)\.([a-z])/).count > 0
       code.gsub!(/(\d)\.([a-z])/, $1 + $2)
     end
-    standard["ccssiSimplifiedDotNotation"] = code
+    standard["CCSS"]["simplifiedDotNotation"] = code
   end
 
   def add_jurisdiction(standard)
@@ -108,4 +89,17 @@ def convert_grade_levels_to_ceds(standard)
         ASN_TO_CEDS_LEVELS[value]
     end
   end 
+
+  def standards_without_codes(standard)
+    # pull out standards without codes
+    return if standard.has_key?("ccssiSimplifiedDotNotation")
+    #code = ask "What is the ccssi sipmlified dot notation for #{hash.statement}?"
+    #flat[i]["ccssiSimplifiedDotNotation"] = code
+    #ret = standard
+  end
+
+  def add_grade_level(standard)
+    standard["gradeLevel"] = GRADES[standard["gradeLevels"]]
+  end
+
 end

chisel/lib/chisel/ccssi_identifier.rb

@@ -42,6 +42,18 @@ def doc
     @doc ||=  MultiJson.decode(file)
   end
 
+  def math
+      doc.delete_if do |standard|
+         standard['dot_notation'].match(/Math/).nil?
+      end
+  end
+
+  def literacy
+      doc.delete_if do |standard|
+        standard['dot_notation'].match(/Literacy/).nil?
+      end
+  end
+
   def self.combine_v1_with_v2(doc_1, doc_2)
     doc_2.each_with_index do |standard, i|
       doc_2[i]['dot_notation_v1'] = find_standard_by_guid(doc_1, standard['GUID'])['dot_notation']

chisel/lib/chisel/conversions.rb

@@ -1,16 +1,41 @@
 
-grades = {
+GRADES = {
  "K" => "Kindergarten",
+ "KG" => "Kindergarten",
  "1" => "Grade 1",
+ "01" => "Grade 1",
  "2" => "Grade 2",
+ "02" => "Grade 2",
  "3" => "Grade 3",
+ "03" => "Grade 3",
  "4" => "Grade 4",
+ "04" => "Grade 4",
  "5" => "Grade 5",
+ "05" => "Grade 5",
  "6" => "Grade 6",
+ "06" => "Grade 6",
  "7" => "Grade 7",
+ "07" => "Grade 7",
  "8" => "Grade 8",
+ "08" => "Grade 8",
+ "09" => "Grade 9",
+ "10" => "Grade 10",
+ "11" => "Grade 11",
+ "12" => "Grade 12",
  "HS" => "High School",
- "MP" => "K-12"
+ "MP" => "K-12",
+ ["KG", "01"] => "Kindergarten - Grade 1",
+ ["KG", "01", "02"] => "Kindergarten - Grade 2",
+ ["KG", "01", "02", "03"] => "Kindergarten - Grade 3",
+ ["01", "02", "03", "04", "05"] => "Grades 1-5",
+ ["03", "04", "05"] => "Grades 3-5",
+ ["04", "05", "06", "07", "08", "09", "10", "11", "12"] => "Grades 4-12",
+ ["06", "07", "08"] => "Grades 6-8",
+ ["06", "07", "08", "09", "10", "11", "12"] => "Grades 6-12",
+ ["09", "10", "11", "12"] => "Grades 9-12",
+ ["09", "10"] => "Grades 9-10",
+ ["11", "12"] => "Grades 11-12",
+ ["KG", "01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12"] => "K-12"
 }
 
 
@@ -86,7 +111,7 @@
 "8"=> "Eighth Grade",
 "9" => "Ninth Grade",
 "10" => "Tenth Grade",
-"11" =>  "Eleventhh Grade",
+"11" =>  "Eleventh Grade",
 "12"=> "Twelfth Grade"
 }