Refactor debezium demo towards a perf oriented version. (deephaven#1969)

* Refactoring to allow separation of demo and perf; add a command interface to loadgen.

* Various fixes.

* Allow command interface to be visible from the host.

debezium-demo/.env → debezium/demo/.env

@@ -1,4 +1,4 @@
-COMPOSE_PROJECT_NAME=core-debezium
+COMPOSE_PROJECT_NAME=core-debezium-demo
 REPO=ghcr.io/
 TAG=main
 PORT=10000

debezium-demo/README.md → debezium/demo/README.md

@@ -10,9 +10,11 @@ The demo follows closely the one defined for Materialize here:
 https://github.com/MaterializeInc/ecommerce-demo/blob/main/README_RPM.md
 
 The load generation script is in `loadgen/generate_load.py`.
-At the beginning of the script the constant `purchaseGenEveryMS`
-controls how frequently purchases are triggered.
 
+It is possible to configure the update rate for both purchase
+(mysql updates) and pageviews (kafka pageview events) via
+ENVIRONMENT arguments set for the loadgen image in the docker-compose.yml
+file.
 
 How to run
 ==========
@@ -27,8 +29,8 @@ Start docker-compose with the compose file in this
 directory:
 
 ```
-cd debezium-demo
-docker-compose up
+cd debezium/demo
+docker-compose up --build
 ```
 
 Then start a Deephaven web console (will be in python mode

debezium/demo/docker-compose.yml

@@ -0,0 +1,70 @@
+# docker compose file to run the debezium-kafka ecommerce demo
+# with DHC ticking tables and dashboard
+
+version: '3.4'
+
+services:
+  server:
+    extends:
+      file: ../../docker-compose-common.yml
+      service: server
+
+  web:
+    extends:
+      file: ../../docker-compose-common.yml
+      service: web
+
+  # Should only be used for non-production deployments, see grpc-proxy/README.md for more info
+  grpc-proxy:
+    extends:
+      file: ../../docker-compose-common.yml
+      service: grpc-proxy
+    depends_on:
+      server:
+        condition: service_healthy
+
+  envoy:
+    # A reverse proxy configured for no SSL on localhost. It fronts the requests
+    # for the static content and the websocket proxy.
+    extends:
+      file: ../../docker-compose-common.yml
+      service: envoy
+    depends_on:
+      server:
+        condition: service_healthy
+      grpc-proxy:
+        condition: service_started
+      web:
+        condition: service_started
+
+  redpanda:
+    extends:
+      file: ../docker-compose-debezium-common.yml
+      service: redpanda
+
+  mysql:
+    extends:
+      file: ../docker-compose-debezium-common.yml
+      service: mysql
+
+  debezium:
+    extends:
+      file: ../docker-compose-debezium-common.yml
+      service: debezium
+    depends_on:
+      redpanda:
+        condition: service_started
+
+  loadgen:
+    build: ../loadgen
+    environment:
+      - PAGEVIEWS_PER_SECOND_START=750
+      - PURCHASES_PER_SECOND_START=10
+      - COMMAND_ENDPOINT=0.0.0.0:8090
+    depends_on:
+      mysql:
+        condition: service_started
+      debezium:
+        condition: service_started
+    ports:
+      - "127.0.0.1:8090:8090"

debezium-demo/docker-compose.yml → debezium/docker-compose-debezium-common.yml

@@ -4,39 +4,6 @@
 version: '3.4'
 
 services:
-  server:
-    extends:
-      file: ../docker-compose-common.yml
-      service: server
-
-  web:
-    extends:
-      file: ../docker-compose-common.yml
-      service: web
-
-  # Should only be used for non-production deployments, see grpc-proxy/README.md for more info
-  grpc-proxy:
-    extends:
-      file: ../docker-compose-common.yml
-      service: grpc-proxy
-    depends_on:
-      server:
-        condition: service_healthy
-
-  envoy:
-    # A reverse proxy configured for no SSL on localhost. It fronts the requests
-    # for the static content and the websocket proxy.
-    extends:
-      file: ../docker-compose-common.yml
-      service: envoy
-    depends_on:
-      server:
-        condition: service_healthy
-      grpc-proxy:
-        condition: service_started
-      web:
-        condition: service_started
-
   redpanda:
     image: docker.vectorized.io/vectorized/redpanda:v21.9.5
     command:
@@ -78,16 +45,5 @@ services:
       VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
       CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: http://redpanda:8081
       CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://redpanda:8081
-    depends_on:
-      redpanda:
-        condition: service_started
     ports:
       - 8083:8083
-
-  loadgen:
-    build: loadgen
-    depends_on:
-      mysql:
-        condition: service_started
-      debezium:
-        condition: service_started

debezium-demo/loadgen/Dockerfile → debezium/loadgen/Dockerfile

@@ -1,8 +1,10 @@
 FROM python:3
 
+EXPOSE 8090
+
 RUN apt-get update && apt-get -qy install curl
 
-RUN pip install barnum kafka-python mysql-connector-python requests noise wait-for-it
+RUN pip install barnum kafka-python mysql-connector-python requests wait-for-it
 
 COPY . /loadgen