feat: updated config (pydantic), refactoring code

.env.example

@@ -0,0 +1,17 @@
+EXPORTER_MONITORING__METRICS_ENABLED=false
+EXPORTER_MONITORING__METRICS_HOST=localhost
+EXPORTER_MONITORING__METRICS_PORT=8125
+EXPORTER_MONITORING__SENTRY_ENABLED=false
+EXPORTER_MONITORING__SENTRY_DSN=https://[email protected]/1
+
+EXPORTER_CLICKHOUSE__ENABLE_UPLOAD=true
+EXPORTER_CLICKHOUSE__HOST=localhost
+EXPORTER_CLICKHOUSE__PORT=8443
+EXPORTER_CLICKHOUSE__PROTO=https
+EXPORTER_CLICKHOUSE__CACERT_PATH=/etc/ssl/ca.pem
+EXPORTER_CLICKHOUSE__USERNAME=tracker
+EXPORTER_CLICKHOUSE__PASSWORD=mypassword
+EXPORTER_CLICKHOUSE__DATABASE=tracker
+EXPORTER_CLICKHOUSE__SERVERLESS_PROXY_ID=xxxxxxxxxxxxxxxx
+EXPORTER_CLICKHOUSE__ISSUES_TABLE=issues
+EXPORTER_CLICKHOUSE__ISSUE_METRICS_TABLE=issue_metrics

.github/workflows/docker.yaml

@@ -0,0 +1,5 @@
+name: Docker
+on:
+  push:
+    tags:
+      - '*'

.github/workflows/pypi.yaml

@@ -0,0 +1,33 @@
+name: PyPI
+on:
+  push:
+    tags:
+      - '*'
+
+env:
+  PYPI_FROM_GITHUB: 1
+
+jobs:
+  build:
+    name: Build & Push PyPI package
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744
+      - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1
+        with:
+          python-version: '3.10.x'
+          cache: 'pip'
+          cache-dependency-path: '**/requirements*.txt'
+      - run: pip install -r requirements-dev.txt
+      - name: Extract tag name
+        id: tag
+        run: echo ::set-output name=TAG_NAME::$(echo $GITHUB_REF | cut -d / -f 3)
+      - name: Update version in setup.py
+        run: >-
+          sed -i "s/{{PKG_VERSION}}/${{ steps.tag.outputs.TAG_NAME }}/g" setup.py
+      - name: Build a binary wheel
+        run: make dist
+      - name: Publish distribution to PyPI
+        uses: pypa/gh-action-pypi-publish@master
+        with:
+          password: ${{ secrets.PYPI_API_TOKEN }}

.github/workflows/tests.yaml

@@ -0,0 +1,73 @@
+name: Tests
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - "tracker_exporter/**"
+      - "tests/**"
+    branches:
+      - master
+  push:
+    paths:
+      - "tracker_exporter/**"
+      - "tests/**"
+    branches:
+      - master
+  schedule:
+    - cron: '20 4 * * 6'
+
+env:
+  EXPORTER_TRACKER__TOKEN: ${{ secrets.EXPORTER_TRACKER__TOKEN }}
+  EXPORTER_TRACKER__CLOUD_ORG_ID: ${{ secrets.EXPORTER_TRACKER__CLOUD_ORG_ID }}
+
+jobs:
+  pytest:
+    name: pytest
+    runs-on: ${{matrix.os}}
+    strategy:
+      matrix:
+        python-version:
+          - "3.10"
+        os:
+          - ubuntu-latest
+          - windows-latest
+          - macos-latest
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: 'pip'
+          cache-dependency-path: '**/requirements*.txt'
+      - name: Install dependencies
+        run: |
+          python -W ignore -m pip install --upgrade pip
+          python -W ignore -m pip install -U pytest-cov
+          python -W ignore -m pip install -r requirements.txt
+          python -W ignore -m pip install -r requirements-dev.txt
+          python -W ignore -m pip install pytest-xdist[psutil]
+
+      - name: Test with pytest
+        run: |
+          pytest -vv --cov=tracker_exporter --cov-append -n auto --junit-xml=.test_report.xml
+        env:
+          JOB_INDEX: ${{ strategy.job-index }}
+
+      - name: Test Summary
+        id: test_summary
+        uses: test-summary/[email protected]
+        if: always()  # always run, even if tests fail
+        with:
+          paths: |
+            .test_report.xml
+
+      - name: Submit coverage
+        uses: codecov/codecov-action@v3
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+        with:
+          env_vars: OS,PYTHON
+          name: ${{ matrix.os }}-${{ matrix.python-version }}
+          fail_ci_if_error: true

.gitignore

@@ -1,4 +1,9 @@
 scripts/*
+clickhouse
+migrate
+test.json
+state.json
+.ruff*
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

Dockerfile

@@ -1,15 +1,18 @@
-FROM python:3.7-slim
+FROM python:3.10-slim
 LABEL maintainer="[email protected]"
 LABEL name="tools/tracker-exporter"
 
 ENV DEBIAN_FRONTEND noninteractive
-
-# Configure timezone
-RUN apt-get -qq update
-RUN apt-get install -yqq tzdata
 ENV TZ=Europe/Moscow
-RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
-RUN dpkg-reconfigure -f noninteractive tzdata
+
+# Prepare environment & packages
+RUN apt-get -qq update && \
+    apt-get install -yqq tzdata && \
+    ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && \
+    echo $TZ > /etc/timezone && \
+    dpkg-reconfigure -f noninteractive tzdata && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
 
 # Configure exporter
 RUN mkdir -p /opt/exporter

Makefile

@@ -1,15 +1,36 @@
-.PHONY: clean clean-build clean-pyc dist help
+.PHONY: clean clean-build clean-pyc dist help clickhouse test tests migration docs
 .DEFAULT_GOAL := help
 
 help:
-	@echo "clean - remove all artifacts"
-	@echo "clean-build - remove build artifacts"
-	@echo "clean-pyc - remove python artifacts"
-	@echo "install - install the package"
-	@echo "init - initialize the development environment"
-	@echo "dist - build package"
-	@echo "release - upload package to PyPi"
-	@echo "lint - check style with pylint"
+	@echo "🪄  PREPARE ENVIRONMENT"
+	@echo "---------------------------------------------------------------------"
+	@echo "  init                Install all python requirements"
+	@echo "  pre-commit          Install pre-commit hooks"
+	@echo ""
+	@echo "👀  CHECK"
+	@echo "---------------------------------------------------------------------"
+	@echo "  test                Run tests (pytest)"
+	@echo "  test-no-cov         Run tests (pytest) without coverage report"
+	@echo "  lint                Check python syntax & style by pylint"
+	@echo "  sec                 Security linter (bandit)"
+	@echo ""
+	@echo "🛠  INSTALL & RELEASE"
+	@echo "---------------------------------------------------------------------"
+	@echo "  install             Install library to site-packages"
+	@echo "  build               Build package"
+	@echo "  build-docker        Build docker image"
+	@echo "  release             Build & push package to PyPI"
+	@echo "  clean               Clean build/install artifacts"
+	@echo ""
+	@echo "🐳  DEV & RUN"
+	@echo "---------------------------------------------------------------------"
+	@echo "  up                  Up docker composition with app & clickhouse"
+	@echo "  up-clickhouse       Up docker clickhouse"
+	@echo "  down                Down docker composition (full)"
+	@echo "  down-clickhouse     Down docker clickhouse"
+	@echo "  clickhouse          Clickhouse CLI"
+	@echo "  migration           Run clickhouse migration"
+	@echo "  run                 Run ETL"
 
 clean: clean-build clean-pyc
 
@@ -27,19 +48,56 @@ clean-pyc:
 	find . -name '*~' -exec rm -f {} +
 	find . -name '__pycache__' -exec rm -rf {} +
 
+test:
+	@pytest -vv --cov=tracker_exporter
+
+tests: test
+
+test-no-cov:
+	@pytest -v
+
 lint:
-	pylint --max-line-length=120 --rcfile=setup.cfg tracker_exporter
+	@pylint --max-line-length=120 --rcfile=setup.cfg tracker_exporter
+
+sec:
+	@bandit -r tracker_exporter
 
-dist:
-	python3 setup.py sdist bdist_wheel
+build:
+	@python3 setup.py sdist bdist_wheel
 
-release: clean dist
+build-docker:
+	@docker build . -t tracker_exporter:dev
+
+release: clean build
+	@make clean
+	@make build
+	@python3 -m twine upload --repository pypi dist/*
 	@make clean
-	@make dist
-	python3 -m twine upload --repository pypi dist/*
 
 install: clean
-	python3 setup.py install
+	@python3 setup.py install
 
 init:
-	pip3 install -r requirements-dev.txt
+	@pip3 install -r requirements.txt
+	@pip3 install -r requirements-dev.txt
+
+up:
+	@docker compose -f docker-compose.dev.yml up -d
+
+up-clickhouse:
+	@docker compose -f docker-compose.dev.yml up -d clickhouse
+
+down:
+	@docker compose -f docker-compose.dev.yml down
+
+down-clickhouse:
+	@docker compose -f docker-compose.dev.yml down clickhouse
+
+clickhouse:
+	@docker exec -it clickhouse clickhouse-client
+
+run:
+	@tracker-exporter --env-file .env
+
+migration:
+	@./data-migrate.sh