(feat) add local-cache to scully (scullyio#1208)

* fixed repeating tests, move to node 14 for scully
* feat(scully): add local-cache plugin
* fix(monorepo): update tests to use not use the cahce while serving!
* fix(scully): add nocache option to background server

.eslintrc.json

@@ -0,0 +1,31 @@
+{
+  "root": true,
+  "ignorePatterns": ["**/*"],
+  "plugins": ["@nrwl/nx"],
+  "overrides": [
+    {
+      "files": ["*.ts", "*.tsx", "*.js", "*.jsx"],
+      "rules": {
+        "@nrwl/nx/enforce-module-boundaries": [
+          "error",
+          {
+            "enforceBuildableLibDependency": true,
+            "allow": [],
+            "depConstraints": [{ "sourceTag": "*", "onlyDependOnLibsWithTags": ["*"] }]
+          }
+        ]
+      }
+    },
+    {
+      "files": ["*.ts", "*.tsx"],
+      "extends": ["plugin:@nrwl/nx/typescript"],
+      "parserOptions": { "project": "./tsconfig.*?.json" },
+      "rules": {}
+    },
+    {
+      "files": ["*.js", "*.jsx"],
+      "extends": ["plugin:@nrwl/nx/javascript"],
+      "rules": {}
+    }
+  ]
+}

.github/workflows/nodejs.yml

@@ -23,9 +23,9 @@ jobs:
       - run: npm i
       - run: npm run symlinks
       - run: npx nx run-many --target=build --all --prod
-      - run: node ./dist/libs/scully/src/scully --tds --project=sample-blog
-      - run: node ./dist/libs/scully/src/scully killServer --project=scully-docs
-      - run: node ./dist/libs/scully/src/scully --project=scully-docs
+      - run: node ./dist/libs/scully/src/scully --tds --project=sample-blog --noCache
+      - run: node ./dist/libs/scully/src/scully killServer --project=scully-docs  --noCache
+      - run: node ./dist/libs/scully/src/scully --project=scully-docs  --noCache
       - run: npm run jest:test
       - run: npx cypress install
       - run: npm run cypress:e2e:server

.vscode/extensions.json

@@ -1,6 +1,3 @@
 {
-  "recommendations": [
-    "ms-vscode.vscode-typescript-tslint-plugin",
-    "esbenp.prettier-vscode"
-  ]
+  "recommendations": ["ms-vscode.vscode-typescript-tslint-plugin", "esbenp.prettier-vscode", "firsttris.vscode-jest-runner"]
 }

jest.config.js

@@ -1,19 +1,20 @@
 module.exports = {
   projects: [
-    '<rootDir>/libs/scully',
-    '<rootDir>/libs/ng-lib',
-    '<rootDir>/apps/sample-blog',
-    '<rootDir>/apps/scully-docs',
-    '<rootDir>/libs/plugins/extra',
+    // '<rootDir>/libs/scully',
+    // '<rootDir>/libs/ng-lib',
+    // '<rootDir>/apps/sample-blog',
+    // '<rootDir>/apps/scully-docs',
+    // '<rootDir>/libs/plugins/extra',
     '<rootDir>/libs/plugins/from-data',
-    '<rootDir>/libs/plugins/base-href-rewrite',
-    '<rootDir>/libs/plugins/scully-plugin-flash-prevention',
-    '<rootDir>/libs/plugins/logrocket',
-    '<rootDir>/libs/plugins/sentry',
-    '<rootDir>/libs/plugins/google-analytics',
-    '<rootDir>/libs/plugins/docs-link-update',
-    '<rootDir>/libs/plugins/scully-plugin-remove-scripts',
-    '<rootDir>/libs/plugins/scully-plugin-critical-css',
-    '<rootDir>/libs/plugins/scully-plugin-copy-to-clipboard',
+    // '<rootDir>/libs/plugins/scully-plugin-local-cache',
+    // '<rootDir>/libs/plugins/base-href-rewrite',
+    // '<rootDir>/libs/plugins/scully-plugin-flash-prevention',
+    // '<rootDir>/libs/plugins/logrocket',
+    // '<rootDir>/libs/plugins/sentry',
+    // '<rootDir>/libs/plugins/google-analytics',
+    // '<rootDir>/libs/plugins/docs-link-update',
+    // '<rootDir>/libs/plugins/scully-plugin-remove-scripts',
+    // '<rootDir>/libs/plugins/scully-plugin-critical-css',
+    // '<rootDir>/libs/plugins/scully-plugin-copy-to-clipboard',
   ],
 };

jest.preset.js

@@ -1,16 +1,4 @@
-const nxPreset = require('@nrwl/jest/preset');
-// module.exports = {
-//   testMatch: ['**/+(*.)+(spec|test).+(ts|js)?(x)'],
-//   transform: {
-//     '^.+\\.(ts|js|html)$': 'ts-jest'
-//   },
-//   resolver: '@nrwl/jest/plugins/resolver',
-//   moduleFileExtensions: ['ts', 'js', 'html'],
-//   coverageReporters: ['html']
-// };
 module.exports = {
-  ...nxPreset,
-  // preset: 'jest-preset-angular',
   roots: ['tests/jest/src'],
   transform: {
     '^.+\\.(ts|js|html)$': 'ts-jest',

libs/plugins/scully-plugin-local-cache/.eslintrc.json

@@ -0,0 +1,5 @@
+{
+  "extends": "../../../.eslintrc.json",
+  "ignorePatterns": ["!**/*"],
+  "rules": {}
+}

libs/plugins/scully-plugin-local-cache/README.md

@@ -0,0 +1,101 @@
+# scully-plugin-local-cache
+
+> ## note
+>
+> This plugin is in BETA, please let us know any issues you have with it
+
+This is the home of Scully local cache plugin for Scully.
+It provides local caching for Scully.
+
+The cache works with a approximately 5 minute granularity. This means that when a resource is in the cache it will live for the time it has assigned to it, but with a 5 minute margin.
+When you have resources where this 5 minutes cache time is to long, set the resource to a TTL of `0` making it un-cachable.
+
+To use this plugin you must first install it using npm:
+
+```bash
+ npm i @scullyIO/scully-plugin-local-cache
+```
+
+After that add it to your config file:
+
+```typescript
+import { localCacheReady } from '@scullyIO/scully-plugin-local-cache';
+import { ScullyConfig } from '@scullyio/scully';
+
+/** note that the config must be a promise */
+export const config: Promise<ScullyConfig> = (async () => {
+
+// setPluginConfig(theVault, { customerId: 2, projectId: 2 });
+
+/** Mandatory! *//
+await localCacheReady({
+   includeReferer: false,
+ });
+
+ const conf: ScullyConfig = {
+   projectRoot: './apps/demo/src',
+   projectName: 'demo',
+   outDir: './dist/static',
+   handle404: 'index',
+   maxRenderThreads: 24,
+   routes: {
+     '/user/:id': {
+       type: 'json',
+       id: {
+         url: 'http://localhost:8200/users',
+         resultsHandler: (raw) => raw.slice(0, 20),
+         property: 'id',
+       },
+     },
+   },
+ };
+ return conf;
+})();
+```
+
+## config must be a promise.
+
+Because the local-cache needs to initialize its cache before it can be used, and this is an asynchronous process, this plugin demands the Scully config the return a promise. You can use an async function as showen in the above config, or use a normal promise chain when that is your preference.
+
+## Runtime Parameters
+
+### nocache
+
+If you want to do a local run of your app, without the Vault being used, you can add the command line option `--noCache` to your scully command.
+
+```bash
+npx scully --project MyProject --noCache
+```
+
+When you want to run a separate scully server, it _must_ exclude the cache. start it like this:
+
+```bash
+npx scully --project MyProject serve --noCache
+```
+
+This is needed, because the _local_ cache can only be used by once Scully instance.
+
+### clearCache
+
+the `--clearCache` option removes the local cache.
+
+## The Vault Config
+
+The local-cache has a config object that can be set using `setPluginConfig(theVault, customConfig);` or using the `await theVaultReady(customConfig);` option.
+Settings provided to the `theVaultReady()` function will overwrite the `setPluginConfig`
+
+```typescript
+export interface LocalCacheConfig {
+  /** use the Referer as a key differentiation, defaults to false */
+  includeReferer?: boolean;
+  /** your environment */
+  environment?: 'dev' | 'prod' | 'staging' | 'test';
+  /** the default Time To Live. 12 hours if unset */
+  defaultTTL?: number;
+  /** TTL exceptions */
+  ttlExceptions?: {
+    /** the full URL, is used as: `testUrl.startsWith(urlStart)`  */
+    [urlStart: string]: number;
+  };
+}
+```

libs/plugins/scully-plugin-local-cache/jest.config.js

@@ -0,0 +1,14 @@
+module.exports = {
+  displayName: 'local-cache',
+  preset: '../../../jest.preset.js',
+  globals: {
+    'ts-jest': {
+      tsConfig: '<rootDir>/tsconfig.spec.json',
+    },
+  },
+  transform: {
+    '^.+\\.[tj]sx?$': 'ts-jest',
+  },
+  moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx'],
+  coverageDirectory: '../../../coverage/libs/plugins/scully-plugin-local-cache',
+};

libs/plugins/scully-plugin-local-cache/package.json

@@ -0,0 +1,17 @@
+{
+  "name": "@scullyio/scully-plugin-local-cache",
+  "version": "0.0.1",
+  "dependencies": {
+    "level": "*"
+  },
+  "main": "src/index.js",
+  "devDependencies": {
+    "@types/level-js": "^4.0.1"
+  },
+  "peerDependencies": {
+    "rxjs": "*",
+    "yargs": "*",
+    "puppeteer": "*",
+    "@scullyio/scully": "*"
+  }
+}

libs/plugins/scully-plugin-local-cache/src/index.ts

@@ -0,0 +1,2 @@
+export { localCacheReady, localCache } from './lib/plugins-local-cache';
+export { levelDbReady, del, get, set, initializeLevelDb } from './lib/ldb';

libs/plugins/scully-plugin-local-cache/src/lib/Deferred.ts

@@ -0,0 +1,8 @@
+export class Deferred<T> {
+  resolve: (T?) => void;
+  reject: (error?: any) => void;
+  promise = new Promise<T>((rs, rj) => {
+    this.resolve = rs;
+    this.reject = rj;
+  });
+}

libs/plugins/scully-plugin-local-cache/src/lib/cacheDone.ts

@@ -0,0 +1,35 @@
+/* eslint-disable no-prototype-builtins */
+import { log, stopProgress, yellow } from '@scullyio/scully';
+import { usageStatistics as stats } from './usageStatistics';
+
+export const localCacheDone = async () => {
+  /**  */
+  // await logToFile('{}]');
+  stopProgress();
+
+  log(`
+--------------------------------------------------------
+  Local cache statistics:
+--------------------------------------------------------
+  requests: ${yellow(stats.requests.toLocaleString('en-us').padStart(8, ' '))}
+  hits:     ${yellow(stats.hit.toLocaleString('en-us').padStart(8, ' '))}
+  misses:   ${yellow(stats.mis.toLocaleString('en-us').padStart(8, ' '))}
+
+  Traffic over wire: ${yellow(
+    Math.floor(stats.traffic / 1024 / 1024)
+      .toLocaleString('en-us')
+      .padStart(8, ' ')
+  )}Mb
+
+  hit ratio: ${yellow(
+    Math.ceil(stats.hit / (stats.requests / 100))
+      .toLocaleString('en-us')
+      .padStart(8, ' ')
+  )}%
+  mis ratio: ${yellow(
+    Math.floor(stats.mis / (stats.requests / 100))
+      .toLocaleString('en-us')
+      .padStart(8, ' ')
+  )}%
+--------------------------------------------------------`);
+};

libs/plugins/scully-plugin-local-cache/src/lib/cli-options.ts

@@ -0,0 +1,22 @@
+import { command } from 'yargs';
+
+const options = command('cache', 'control cache settings', {
+  cacheClear: {
+    alias: ['cc', 'cache-clear', 'clear-cache', 'cacheClear', 'clearCache'],
+    boolean: true,
+    default: false,
+    description: 'Clear te complete cache ',
+  },
+  list: {
+    boolean: true,
+    default: false,
+    description: 'list local cache content urls',
+  },
+  noCache: {
+    alias: ['nc', 'no-cache'],
+    boolean: true,
+    default: false,
+    description: 'do not use caching',
+  },
+}).help().argv;
+export const { list, cacheClear, noCache } = options;

libs/plugins/scully-plugin-local-cache/src/lib/config.ts

@@ -0,0 +1,28 @@
+import { getMyConfig, logWarn } from '@scullyio/scully';
+import { LocalCacheConfig } from './local-cache.interface';
+import { localCache } from './plugins-local-cache';
+
+/** this is the "global" config for theVault, default values are set here */
+export const config: Required<LocalCacheConfig> = {
+  includeReferer: false,
+  environment: 'dev',
+  /** 12 hours. */
+  defaultTTL: 12 * 60 * 60 * 1000,
+  ttlExceptions: {},
+};
+
+/**
+ * Update the global config, takes a config object as param
+ * wil use above global, and merge in the pluginConfig and the param.
+ * Param takes precedence
+ * @param configUpdate
+ */
+export function updateConfig(configUpdate: LocalCacheConfig) {
+  /** merge in user settings */
+  const userConfig = {
+    ...config,
+    ...(getMyConfig(localCache) as LocalCacheConfig),
+    ...configUpdate,
+  };
+  Object.entries(userConfig).forEach(([key, val]) => (config[key] = val));
+}

libs/plugins/scully-plugin-local-cache/src/lib/handlePuppeteerRequest.ts

@@ -0,0 +1,46 @@
+/* eslint-disable no-prototype-builtins */
+import { Request } from 'puppeteer';
+import { config } from './config';
+import { del, get } from './ldb';
+import { usageStatistics } from './usageStatistics';
+import { CacheItem } from './local-cache.interface';
+
+export async function handlePuppeteerRequest(request: Request) {
+  const url = request.url();
+  const { referer, ...headers } = request.headers();
+  if (config.includeReferer) {
+    headers.referer = referer;
+  }
+  usageStatistics.requests += 1;
+  const hash: string = await get<string>({ url, headers }).catch(() => '');
+
+  if (hash) {
+    const cache: CacheItem = await get<CacheItem>({ hash }).catch(() => undefined);
+    if (isValid(cache)) {
+      usageStatistics.hit += 1;
+      return request.respond(cache.response);
+    }
+  }
+  usageStatistics.mis += 1;
+  request.continue();
+  return;
+}
+
+export function isValid(cache: CacheItem): boolean {
+  try {
+    if (!cache.hasOwnProperty('TTL')) {
+      return false;
+    }
+    const now = Date.now();
+    if (cache.inserted + cache.TTL > now) {
+      return true;
+    } else {
+      /** this isn't a valid cache item anymore, remove */
+      del({ hash: cache.hash });
+    }
+  } catch {
+    /** placeholder */
+  }
+
+  return false;
+}