Document Schema File Naming Conventions

.circleci/config.yml

@@ -7,6 +7,7 @@ references:
       OSCAL_REPO_DIR: git-oscal
       OSCAL_BUILD_DIR: build_artifacts
       SCHEMATRON_HOME: git-schematron
+      TERM: xterm
     working_directory: ~/oscal
   attach_build_workspace: &attach_build_workspace
     attach_workspace:
@@ -63,6 +64,27 @@ commands:
           name: Get ajv
           command: |
             sudo npm install -g ajv-cli
+  install-markdown-link-check:
+    description: "Retrieve markdown-link-check and setup the environment to run it"
+    steps:
+      - run:
+          name: Get markdown-link-check
+          command: |
+            sudo npm install -g markdown-link-check
+  install-jsome:
+    description: "Retrieve jsome and setup the environment to run it"
+    steps:
+      - run:
+          name: Get jsome
+          command: |
+            sudo npm install -g jsome
+  install-prettyjson:
+    description: "Retrieve prettyjson and setup the environment to run it"
+    steps:
+      - run:
+          name: Get prettyjson
+          command: |
+            sudo npm install -g prettyjson
   install-hub:
     description: "Retrieve hub and setup the environment to run it"
     steps:
@@ -190,6 +212,8 @@ jobs: # a collection of
       - *attach_build_workspace
       - check-skip-build
       - install-saxon
+      - install-jsome
+      - install-prettyjson
       - run:
           name: Generate OSCAL converters
           command: |
@@ -199,6 +223,17 @@ jobs: # a collection of
           root: build_artifacts
           paths:
             - content # not needed?
+  job-validate-markdown:
+    executor: java-executor
+    steps:
+      - *checkout
+      - *attach_build_workspace
+      - check-skip-build
+      - install-markdown-link-check
+      - run:
+          name: Validate content instances
+          command: |
+            find "$OSCAL_REPO_DIR" -path ./docs -prune -o -name \*.md -exec markdown-link-check -q {} \;
   job-validate-content:
     executor: java-executor
     steps:
@@ -221,10 +256,10 @@ jobs: # a collection of
       - run:
           name: Generate schema documentation
           command: |
-            cp -r "$OSCAL_BUILD_DIR"/* "$OSCAL_REPO_DIR"
-            bash "$CICD_DIR/generate-schema-documentation.sh" "$OSCAL_BUILD_DIR"
-            echo "-- fix the fact that we cannot generate directly to the build directory due to a bug"
-            mv docs "$OSCAL_BUILD_DIR"/docs
+            #cp -r "$OSCAL_BUILD_DIR"/* "$OSCAL_REPO_DIR"
+            bash -x "$CICD_DIR/generate-model-documentation.sh" "$OSCAL_BUILD_DIR"
+            # echo "-- fix the fact that we cannot generate directly to the build directory due to a bug"
+            # mv docs "$OSCAL_BUILD_DIR"/docs
       - *publish_artifacts
       - persist_to_workspace:
           root: build_artifacts
@@ -245,8 +280,9 @@ jobs: # a collection of
             git rm -r --ignore-unmatch xml/schema/*.xsd
             git rm -r --ignore-unmatch json/convert/*.xsl
             git rm -r --ignore-unmatch json/schema/*.json
-            git rm -r --ignore-unmatch content/**/*.json
             git rm -r --ignore-unmatch content/**/*.xml
+            git rm -r --ignore-unmatch content/**/*.json
+            git rm -r --ignore-unmatch content/**/*.yaml
             # Copy new built files to repo
             cd -
             cp -r "$OSCAL_BUILD_DIR"/* "$OSCAL_REPO_DIR"
@@ -323,6 +359,14 @@ workflows:
   build:
     jobs:
       - job-ci-skip-check
+      - job-validate-markdown:
+          requires:
+            - job-ci-skip-check
+          filters:
+            branches:
+              ignore:
+                - master
+                - nist-pages
       - job-validate-metaschema:
           requires:
             - job-ci-skip-check

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,15 +1,15 @@
 # Committer Notes
 
-{Please provide a brief description of what this PR accomplishes. Be sure to reference any issues addressed. If the PR is a work-in-progress submitted for early review, please include [WIP] at the beginning of the title.}
+{Please provide a brief description of what this PR accomplishes. Be sure to reference any issues addressed. If the PR is a work-in-progress submitted for early review, please include [WIP] at the beginning of the title or mark the PR as DRAFT.}
 
 ### All Submissions:
 
-* [ ] Have you followed the guidelines in our [Contributing](https://github.com/usnistgov/OSCAL/blob/master/CONTRIBUTING.md) document?
-* [ ] Have you checked to ensure there aren't other open [Pull Requests](../../pulls) for the same update/change?
-* [ ] Have you squashed any non-relevant commits and commit messages? \[[instructions](https://git-scm.com/book/en/v2/Git-Tools-Rewriting-History)\]
+- [ ] Have you followed the guidelines in our [Contributing](https://github.com/usnistgov/OSCAL/blob/master/CONTRIBUTING.md) document?
+- [ ] Have you checked to ensure there aren't other open [Pull Requests](https://github.com/usnistgov/OSCAL/pulls) for the same update/change?
+- [ ] Have you squashed any non-relevant commits and commit messages? \[[instructions](https://git-scm.com/book/en/v2/Git-Tools-Rewriting-History)\]
 
 ### Changes to Core Features:
 
-* [ ] Have you added an explanation of what your changes do and why you'd like us to include them?
-* [ ] Have you written new tests for your core changes, as applicable?
-* [ ] Have you included examples of how to use your new feature(s)?
+- [ ] Have you added an explanation of what your changes do and why you'd like us to include them?
+- [ ] Have you written new tests for your core changes, as applicable?
+- [ ] Have you included examples of how to use your new feature(s)?

.markdown-proofing

@@ -0,0 +1,23 @@
+{
+  "presets": [],
+  "analyzers": [
+    "passive-voice",
+    "require-oxford-commas",
+    "sensitivity",
+    "sentiment",
+    "spelling",
+    "statistics",
+    "write-good"
+  ],
+  "rules": {
+    "missing-oxford-commas": "warning",
+    "passive-voice-usage-count": "info, warning > 2",
+    "sensitivity": "warning",
+    "sentiment-score": "info",
+    "sentiment-comparative-score": "info",
+    "spelling-error": "error",
+    "statistics-flesch-kincaid-grade-level": "info, warning > 12",
+    "statistics-flesch-kincaid-reading-ease": "warning <= 40",
+    "write-good": "info"
+  }
+}

.spelling

@@ -0,0 +1,71 @@
+ad-hoc
+boolean
+bulleted
+catalog
+catalogs
+circleci
+CODE_OF_CONDUCT.md
+conformant
+consensus
+CONTRIBUTING.md
+datatypes
+deliverables
+duplicative
+e.g.
+fedramp
+fedramp.gov
+formatter
+formatters
+github
+http
+i.e.
+ISSUE_TEMPLATE
+json
+json-based
+jekyll
+LICENSE.md
+localhost
+lossless
+metadata
+metaschema
+metaschemas
+namespace
+namespaces
+ncname
+nist.gov
+pipelined
+quickstart
+repo
+rev4
+runtime
+saxon
+saxonhe
+schemas
+schematron
+schematrons
+sexualized
+SP800-53
+src
+stylesheet
+stylesheets
+subcomponent
+subcomponents
+subcontrol
+subcontrols
+sublicensable
+toolchain
+U.S.
+unformatted
+USERS.md
+utf-8
+xml
+xpath
+xproc
+xq
+xquery
+xqueries
+xsd
+xsl
+xslt
+xslts
+

build/ci-cd/common-environment.sh

@@ -8,6 +8,7 @@ OSCALDIR=$(cd "$DIR/../.."; pwd)
 
 ## Setup color codes
 # check if stdout is a terminal...
+export TERM=${TERM:-dumb}
 colorize=0
 if [ -t 1 ]; then
   # does the terminal support colors?

build/ci-cd/config/metaschema

@@ -1,5 +1,4 @@
-# path to metaschema source
-src/metaschema/oscal-catalog-metaschema.xml
-src/metaschema/oscal-profile-metaschema.xml
-src/metaschema/oscal-ssp-metaschema.xml
-
+# path to metaschema source|generate schema format(s)|generate converter target format(s)|generate docs format(s)
+src/metaschema/oscal_catalog_metaschema.xml|xml,json|xml,json|xml,json
+src/metaschema/oscal_profile_metaschema.xml|xml,json|xml,json|xml,json
+src/metaschema/oscal_ssp_metaschema.xml|xml,json|xml,json|

build/ci-cd/copy-and-convert-content.sh

@@ -7,16 +7,17 @@ fi
 
 source $OSCALDIR/build/ci-cd/saxon-init.sh
 
-if [[ -z "$1" ]]; then
-  working_dir=$OSCALDIR
+if [ -z "$1" ]; then
+  working_dir="$OSCALDIR"
 else
-  working_dir=$1
+  working_dir="$1"
 fi
+echo "${P_INFO}Working in '${P_END}${working_dir}${P_INFO}'.${P_END}"
 
 exitcode=0
 shopt -s nullglob
 shopt -s globstar
-while IFS="|" read path format type converttoformats || [[ -n "$path" ]]; do
+while IFS="|" read path format model converttoformats || [[ -n "$path" ]]; do
   shopt -s extglob
   [[ "$path" =~ ^[[:space:]]*# ]] && continue
   # remove leading space
@@ -31,26 +32,44 @@ while IFS="|" read path format type converttoformats || [[ -n "$path" ]]; do
     for file in $files_to_process
     do
       dest="$working_dir/${file/$OSCALDIR\/src\//}"
-      dest_dir=${dest%/*}
+      dest_dir=${dest%/*} # remove filename
       echo "${P_INFO}Copying '$file' to '$dest'.${P_END}"
       mkdir -p "$dest_dir"
       cp "$file" "$dest"
 
       IFS=","
       for altformat in "$converttoformats"; do
-        newpath="${file/$OSCALDIR\/src\//}"
-        newpath="${newpath/\/$format\///$altformat/}"
-        newpath="${newpath%.$format}.$altformat"
-        dest="$working_dir/$newpath"
-        converter="$working_dir/$altformat/convert/oscal-$type-$format-to-$altformat-converter.xsl"
+        newpath="${file/$OSCALDIR\/src\//}" # strip off src
+        newpath="${newpath/\/$format\///$altformat/}" # change path from old to new format dir
+        newpath="${newpath%.*}" # strip extension
 
-        echo "${P_INFO}Generating $altformat file '$dest' from '$file'.${P_END}"
+        dest="$working_dir/${newpath}-min.${altformat}"
+        converter="$working_dir/$altformat/convert/oscal_${model}_${format}-to-${altformat}-converter.xsl"
+
+        echo "${P_INFO}Generating ${altformat^^} file '$dest' from '$file' using converter '$converter'.${P_END}"
         xsl_transform "$converter" "$file" "$dest"
         cmd_exitcode=$?
         if [ $cmd_exitcode -ne 0 ]; then
-          echo "${P_ERROR}Content conversion to $altformat failed for '$file'.${P_END}"
+          echo "${P_ERROR}Content conversion to ${altformat^^} failed for '$file'.${P_END}"
           exitcode=1
         fi
+        # TODO: validate generated file
+
+        case $altformat in
+        json)
+          # produce pretty JSON
+          dest_pretty="$working_dir/${newpath}.${altformat}"
+          jsome -c false -s 2 "$dest" > "$dest_pretty"
+
+          # produce yaml
+          newpath="${newpath/\/json\///yaml/}" # change path 
+          dest_pretty="$working_dir/${newpath}.yaml"
+          dest_pretty_dir=${dest_pretty%/*} # remove filename
+          mkdir -p "$dest_pretty_dir"
+          prettyjson --nocolor=1 --indent=2 --inline-arrays=1 "$dest" > "$dest_pretty"
+          ;;
+        esac
+
       done
     done
   fi