examples: swportal: Add example

Signed-off-by: John Andersen <[email protected]>
spur19 · Dec 9, 2020 · e4e64b8 · e4e64b8
1 parent a09cf00
commit e4e64b8
Show file tree

Hide file tree

Showing 21 changed files with 525 additions and 0 deletions.
diff --git a/.gitignore b/.gitignore
@@ -29,6 +29,7 @@ docs/plugins/dffml_*.rst
 docs/api/**
 docs/changelog.md
 docs/shouldi.md
+docs/swportal.rst
 docs/contributing/consoletest.md
 /consoletest/
 tests/service/logs.txt
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -81,6 +81,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Gitter chatbot tutorial.
 - Option to run dataflow without sources from cli.
 - Sphinx extension for automated testing of tutorials (consoletest)
+- Example of software portal using DataFlows and HTTP service
 ### Changed
 - Renamed `-seed` to `-inputs` in `dataflow create` command
 - Renamed configloader/png to configloader/image and added support for loading JPEG and TIFF file formats

diff --git a/docs/index.rst b/docs/index.rst
@@ -70,6 +70,7 @@ branch can be found `here <master/index.html>`_.
     :caption: Subprojects
 
     shouldi
+    swportal
 
 .. toctree::
     :glob:

diff --git a/examples/swportal/README.rst b/examples/swportal/README.rst
@@ -0,0 +1,94 @@
+Software Portal
+===============
+
+Example of using DFFML to create a web app to discover information about
+software.
+
+All the code for this example project is located under the
+`examples/swportal <https://github.com/intel/dffml/blob/master/examples/swportal/>`_
+directory of the DFFML source code.
+
+Setup
+-----
+
+Install dependencies
+
+.. code-block:: console
+    :test:
+
+    $ python -m pip install -U pip setuptools wheel
+    $ python -m pip install -U dffml dffml-service-http
+    $ python -m pip install -r requirements.txt
+
+Usage
+-----
+
+Run the http service and navigate to http://localhost:8080/
+
+.. warning::
+
+    The ``-insecure`` flag is only being used here to speed up this
+    tutorial. See documentation on HTTP API
+    :doc:`/plugins/service/http/security` for more information.
+
+.. code-block:: console
+    :test:
+    :daemon: 8080
+
+    $ dffml service http server \
+        -port 8080 \
+        -mc-atomic \
+        -mc-config projects \
+        -static html-client \
+        -insecure
+
+Query all projects
+
+.. code-block:: console
+    :test:
+    :replace: cmds[0][-1] = cmds[0][-1].replace("8080", str(ctx["HTTP_SERVER"]["8080"]))
+
+    $ curl -sf http://localhost:8080/projects
+
+Get a specific project. This triggers a project's DataFlow to run.
+
+.. code-block:: console
+    :test:
+    :replace: cmds[0][-1] = cmds[0][-1].replace("8080", str(ctx["HTTP_SERVER"]["8080"]))
+
+    $ curl -sf http://localhost:8080/projects/72b4720a-a547-4ef7-9729-dbbe3265ddaa
+
+Structure
+---------
+
+- The codebase for the client is in `html-client/`
+
+- The DataFlows for each project are in `projects/`
+
+  - Custom operations reside in the `operations/` directory
+
+  - Depdendencies are listed in the top level `requirements.txt` file.
+
+HTML Client
++++++++++++
+
+The website displayed to clients is all vanila HTML, CSS, and JavaScript.
+
+Projects
+++++++++
+
+Each project has a DataFlow which describes how data for the project should be
+collected. Static data can be added directly to the dataflow file. When
+gernating data dynamiclly is required, code can be added to `operations/`.
+
+Notes
+-----
+
+Run a single project's DataFlow from the command line
+
+.. code-block:: console
+    :test:
+
+    $ dffml dataflow run single \
+        -dataflow projects/df/b7cf5596-d427-4ae3-9e95-44be879eae73.yaml \
+        -log debug
diff --git a/examples/swportal/html-client/index.html b/examples/swportal/html-client/index.html
@@ -0,0 +1,47 @@
+<!DOCTYPE html>
+<meta charset="UTF-8">
+<html>
+  <head>
+    <title>Software Portal</title>
+    <script type="text/javascript" src="index.js"></script>
+    <script
+      src="https://unpkg.com/miragejs/dist/mirage-umd.js"
+      crossorigin
+    ></script>
+    <link rel="stylesheet" type="text/css" href="theme.css">
+  </head>
+  <body>
+    <h1>Software Portal</h1>
+
+    <hr />
+
+    <div id="displayProjectContainer" style="display:none">
+      <h3 id="displayProjectName"></h3>
+
+      <table id="displayProjectTable" style="width:400px">
+        <tr>
+          <th>Indicator</th>
+          <th>Status</th>
+        </tr>
+        <tr>
+          <td>Static Analysis</td>
+          <td id="displayProjectStaticAnalysis"></td>
+        </tr>
+        <tr>
+          <td>Legal Compliance</td>
+          <td id="displayProjectLegal"></td>
+        </tr>
+      </table>
+
+      <br />
+      <hr />
+      <br />
+    </div>
+
+    <div>
+      <a href="#" onclick="app.refreshDisplayProject()">Unselect Project</a>
+      <ul id="projectsList"></ul>
+    </div>
+
+  </body>
+</html>
diff --git a/examples/swportal/html-client/index.js b/examples/swportal/html-client/index.js
@@ -0,0 +1,94 @@
+class API {
+  _getOnlyContext(results) {
+    return results[Object.keys(results)[0]];
+  }
+
+  async getProject (projectUUID) {
+    return this._getOnlyContext(await (await fetch('/projects/' + projectUUID)).json());
+  }
+
+  async getProjects () {
+    return this._getOnlyContext(this._getOnlyContext(await (await fetch('/projects')).json()));
+  }
+}
+
+
+class App {
+  constructor(api) {
+    this.api = api;
+    this.elements = {};
+  }
+
+  getElements(root) {
+    var elements = {};
+
+    // Create an object to hold references to all elements, keys are element ids.
+    for (var element of document.querySelectorAll("[id]")) {
+      elements[element.id] = element;
+    }
+
+    return elements;
+  }
+
+  refreshElements(root) {
+    this.elements = this.getElements(root);
+  }
+
+  populateDisplayProject(project) {
+    // Hide the displaed project if there is no data to display
+    if (typeof project === "undefined" || project === null) {
+      this.elements.displayProjectContainer.style.display = "none";
+      return;
+    }
+
+    this.elements.displayProjectContainer.style.display = "block";
+    this.elements.displayProjectName.innerText = project.name;
+    this.elements.displayProjectStaticAnalysis.innerText = project.staticAnalysis;
+    this.elements.displayProjectLegal.innerText = project.legal;
+  }
+
+  async refreshDisplayProject (projectUUID) {
+    if (typeof projectUUID === "undefined" ||
+      projectUUID === null ||
+      projectUUID === "")
+      this.populateDisplayProject();
+    else
+      this.populateDisplayProject(await this.api.getProject(projectUUID));
+  }
+
+  populateProjectsList(projects) {
+    // Clear the list
+    this.elements.projectsList.innerHTML = "";
+    // Create a list element for each project
+    Object.entries(projects).forEach(([uuid, project]) => {
+      var listItem = document.createElement("li");
+      var listItemLink = document.createElement("a");
+
+      this.elements.projectsList.appendChild(listItem);
+      listItem.appendChild(listItemLink);
+
+      listItemLink.innerText = project.name;
+      listItemLink.href = "#" + uuid;
+      listItemLink.onclick = (() => {
+        this.refreshDisplayProject(uuid);
+      });
+    });
+  }
+
+  async refreshProjectsList () {
+    this.populateProjectsList(await this.api.getProjects());
+  }
+}
+
+var app = new App(new API());
+
+window.addEventListener('DOMContentLoaded', async function(event) {
+  // DOM loaded, grab all DOM elements we'll be working with
+  app.refreshElements(document);
+
+  // Get the list of projects
+  app.refreshProjectsList();
+
+  // If there is a project hash, display it
+  app.refreshDisplayProject(window.location.hash.replace("#", ""));
+});
diff --git a/examples/swportal/html-client/theme.css b/examples/swportal/html-client/theme.css
@@ -0,0 +1,12 @@
+.good {
+  background-color: lightgreen;
+}
+
+.dangerous {
+  background-color: red;
+}
+
+table, th, td {
+  border: 1px solid black;
+  border-collapse: collapse;
+}
diff --git a/examples/swportal/operations/__init__.py b/examples/swportal/operations/__init__.py
diff --git a/examples/swportal/operations/definitions.py b/examples/swportal/operations/definitions.py
@@ -0,0 +1,6 @@
+from dffml import Definition
+
+UUID = Definition(name="uuid", primitive="string")
+NAME = Definition(name="name", primitive="string")
+STATIC_ANALYSIS = Definition(name="staticAnalysis", primitive="string")
+LEGEL = Definition(name="legal", primitive="string")
diff --git a/examples/swportal/operations/get_from_http_server.py b/examples/swportal/operations/get_from_http_server.py
@@ -0,0 +1,39 @@
+"""
+This is an example operation of how one might query an API to get the status of
+a project's indicator. We use the httptest module to start an example API
+server.
+"""
+import random
+
+import aiohttp
+import httptest
+from dffml import op, Definition
+
+from .definitions import UUID
+
+
+class ExampleAPIServer(httptest.Handler):
+    def do_GET(self):
+        self.send_response(200)
+        self.send_header("Content-type", "text/plain")
+        self.end_headers()
+        self.wfile.write(random.choice(["PASS", "FAIL"]).encode())
+
+
+@httptest.Server(ExampleAPIServer)
+async def make_request_to_example_server(session, ts=httptest.NoServer()):
+    async with session.get(ts.url()) as resp:
+        return (await resp.read()).decode()
+
+
+@op(
+    inputs={"uuid": UUID},
+    outputs={"result": Definition(name="api_result", primitive="string")},
+    imp_enter={
+        "session": (lambda self: aiohttp.ClientSession(trust_env=True))
+    },
+)
+async def query_an_api(self, uuid: str, ts=httptest.NoServer()) -> str:
+    return {
+        "result": await make_request_to_example_server(self.parent.session)
+    }
diff --git a/examples/swportal/operations/projects.py b/examples/swportal/operations/projects.py
@@ -0,0 +1,23 @@
+import pathlib
+
+import yaml
+from dffml import op, DataFlow
+
+DATAFLOW_DIRECTORY = pathlib.Path(__file__).parent.parent / "projects" / "df"
+EXPORT_SEED_DEFINITIONS = ("uuid", "name")
+NON_PROJECT_DATAFLOWS = ("projects",)
+
+
+@op
+def projects() -> dict:
+    return {
+        path.stem: {
+            i.definition.name: i.value
+            for i in DataFlow._fromdict(
+                **yaml.safe_load(path.read_text())
+            ).seed
+            if i.definition.name in EXPORT_SEED_DEFINITIONS
+        }
+        for path in DATAFLOW_DIRECTORY.glob("*.yaml")
+        if path.stem not in NON_PROJECT_DATAFLOWS
+    }