Added prometheus metrics

Fixes #301
Fortify-Labs · Feb 13, 2021 · 5c04bde · 5c04bde
1 parent 340f06e
commit 5c04bde
Show file tree

Hide file tree

Showing 37 changed files with 1,964 additions and 167 deletions.
diff --git a/devops/kubernetes/src/constructs/fortify/cronjob.ts b/devops/kubernetes/src/constructs/fortify/cronjob.ts
@@ -15,10 +15,14 @@ export interface FortifyCronJobOptions {
 	readonly name: string;
 	readonly version: string;
 
-	// absolute image name
+	/**
+	 * absolute image name
+	 */
 	readonly image?: string;
-	// only the image name, registry and version will be added
-	// will only be used when image variable is not set
+	/**
+	 * only the image name, registry and version will be added
+	 * will only be used when image variable is not set
+	 */
 	readonly imageName?: string;
 
 	readonly env?: EnvVar[] | undefined;
@@ -38,13 +42,14 @@ export class FortifyCronJob extends Construct {
 			? [...options.env, { name: "DEBUG", value: "app::*" }]
 			: [];
 
-		// env.push({ name: "ENVOY_ADMIN_API", value: "http://127.0.0.1:15000" });
-		// env.push({ name: "ISTIO_QUIT_API", value: "http://127.0.0.1:15020" });
-
 		env.push({
 			name: "SENTRY_DSN",
 			value: JOBS_SENTRY_DSN,
 		});
+		env.push({
+			name: "PROMETHEUS_PUSH_GATEWAY",
+			value: "http://prometheus-pushgateway.default:9091",
+		});
 
 		if (!env.find((envvar) => envvar.name === "NODE_ENV")) {
 			env.push({
@@ -89,7 +94,6 @@ export class FortifyCronJob extends Construct {
 											})),
 										],
 										command: [
-											// "scuttle",
 											"npm",
 											"run",
 											"start",

diff --git a/devops/kubernetes/src/constructs/fortify/deployment.ts b/devops/kubernetes/src/constructs/fortify/deployment.ts
@@ -17,7 +17,9 @@ export interface FortifyDeploymentOptions {
 
 	readonly version?: string;
 
-	// Override image incase image shall not be interpolated from name
+	/**
+	 * Override image incase image shall not be interpolated from name
+	 */
 	readonly image?: string;
 
 	readonly replicas?: number;
@@ -102,8 +104,6 @@ export class FortifyDeployment extends Construct {
 			});
 		}
 
-		// env.push({ name: "ENVOY_ADMIN_API", value: "http://127.0.0.1:15000" });
-
 		const image =
 			options.image ??
 			(REGISTRY ?? "") +
@@ -141,7 +141,14 @@ export class FortifyDeployment extends Construct {
 				},
 				revisionHistoryLimit: 3,
 				template: {
-					metadata: { labels },
+					metadata: {
+						annotations: {
+							"prometheus.io/scrape": "true",
+							"prometheus.io/port": "8000",
+							"prometheus.io/path": "/metrics",
+						},
+						labels,
+					},
 					spec: {
 						containers: [
 							{

diff --git a/services/17kmmrbot/.env.example b/services/17kmmrbot/.env.example
@@ -20,3 +20,6 @@ KAFKA_TOPIC=
 KAFKA_FROM_START=
 
 SENTRY_DSN=
+
+LOCAL_METRICS_PORT=8004
+LOCAL_HEALTH_CHECKS=9004
diff --git a/services/17kmmrbot/package-lock.json b/services/17kmmrbot/package-lock.json
diff --git a/services/17kmmrbot/package.json b/services/17kmmrbot/package.json
@@ -1,6 +1,6 @@
 {
   "name": "17kmmrbot",
-  "version": "1.5.4",
+  "version": "1.6.0",
   "description": "Fortify twitch bot",
   "private": "true",
   "engines": {
@@ -48,6 +48,7 @@
     "inversify": "^5.0.5",
     "kafkajs": "^1.15.0",
     "pg": "^8.5.1",
+    "prom-client": "^13.1.0",
     "redis": "^3.0.2",
     "reflect-metadata": "^0.1.13",
     "tmi.js": "^1.7.1",

diff --git a/services/17kmmrbot/src/index.ts b/services/17kmmrbot/src/index.ts
@@ -33,6 +33,8 @@ import { HealthCheck } from "@shared/services/healthCheck";
 
 import { Logger } from "@shared/logger";
 import { Connector } from "@shared/definitions/connector";
+import { MetricsService, servicePrefix } from "@shared/services/metrics";
+import { Gauge, Summary } from "prom-client";
 
 const {
 	KAFKA_FROM_START = "false",
@@ -55,12 +57,30 @@ const {
 	const healthCheck = container.get(HealthCheck);
 	await healthCheck.start();
 
+	const metrics = container.get(MetricsService);
+	await metrics.start();
+
+	const channelsGauge = new Gauge({
+		name: `${servicePrefix}_channels`,
+		help: "Gauge counting the amount of joined twitch channels",
+		registers: [metrics.register],
+	});
+
+	const twitchCommandSummary = new Summary({
+		name: `${servicePrefix}_commands`,
+		help: "Summary of duration & outcomes of invoked twitch commands",
+		registers: [metrics.register],
+		labelNames: ["command", "status"],
+		maxAgeSeconds: 600,
+		ageBuckets: 5,
+	});
+
 	const commands = container.getAll<TwitchCommand>("command");
 	const helpCommand = container.get<TwitchCommand>(HelpCommand);
 
 	const postgres = container.get(PostgresConnector);
 	const userRepo = await postgres.getUserRepo();
-	const channels = await (
+	const channels = (
 		await userRepo.find({
 			select: ["twitchName"],
 			where: { suspended: false },
@@ -69,6 +89,8 @@ const {
 		.map((channel) => channel.twitchName ?? "")
 		.filter((value) => value);
 
+	channelsGauge.set(channels.length);
+
 	const commandProcessor = container.get(BotCommandProcessor);
 	const kafka = container.get(KafkaConnector);
 	const consumer = kafka.consumer({ groupId: KAFKA_GROUP_ID });
@@ -157,6 +179,7 @@ const {
 	});
 
 	client.on("message", async (channel, tags, message, self) => {
+		const end = twitchCommandSummary.startTimer();
 		try {
 			// Ignore echoed messages.
 			if (self) return;
@@ -192,7 +215,13 @@ const {
 						});
 						await command.handler(client, channel, tags, message);
 						transaction.finish();
+
+						end({ command: command.invocations[0], status: 200 });
+					} else {
+						end({ command: command.invocations[0], status: 401 });
 					}
+				} else {
+					end({ status: 404 });
 				}
 			}
 		} catch (e) {
@@ -222,6 +251,7 @@ const {
 				channel,
 				`Something went wrong. (Exception ID: ${exceptionID})`,
 			);
+			end({ status: 500 });
 		}
 	});
 

diff --git a/services/17kmmrbot/src/inversify.config.ts b/services/17kmmrbot/src/inversify.config.ts
@@ -23,6 +23,7 @@ import { CodeCommand } from "./commands/code";
 
 import { SecretsManager } from "@shared/services/secrets";
 import { HealthCheckable } from "@shared/services/healthCheck";
+import { MetricsService } from "@shared/services/metrics";
 import { Connector } from "@shared/definitions/connector";
 
 const container = new Container({ autoBindInjectable: true });
@@ -32,6 +33,8 @@ container.bind(Logger).toSelf().inSingletonScope();
 container.bind(Secrets).toSelf().inSingletonScope();
 container.bind(SecretsManager).toService(Secrets);
 
+container.bind(MetricsService).toSelf().inSingletonScope();
+
 container.bind<TwitchCommand>("command").to(CountdownCommand);
 container.bind<TwitchCommand>("command").to(NotablePlayersCommand);
 container.bind<TwitchCommand>("command").to(DevCommands);

diff --git a/services/17kmmrbot/src/services/command.ts b/services/17kmmrbot/src/services/command.ts
@@ -11,33 +11,63 @@ import {
 } from "@shared/events/systemEvents";
 import { PostgresConnector } from "@shared/connectors/postgres";
 import { convertMS } from "../lib/dateUtils";
+import { Gauge, Summary } from "prom-client";
+import { MetricsService, servicePrefix } from "@shared/services/metrics";
 
 const { BOT_BROADCAST_DISABLED } = process.env;
 
 @injectable()
 export class BotCommandProcessor {
+	kafkaMessageSummary: Summary<"type" | "status">;
+
 	constructor(
 		@inject(PostgresConnector) private postgres: PostgresConnector,
-	) {}
+		@inject(MetricsService) private metrics: MetricsService,
+	) {
+		this.kafkaMessageSummary = new Summary({
+			name: `${servicePrefix}_processed_messages`,
+			help: "Summary of duration & outcomes of processed kafka messages",
+			registers: [this.metrics.register],
+			labelNames: ["type", "status"],
+			maxAgeSeconds: 600,
+			ageBuckets: 5,
+		});
+	}
 
 	async process(payload: EachMessagePayload, client: Client) {
 		const message: FortifyEvent<SystemEventType> = JSON.parse(
 			(payload.message.value ?? "{}").toString(),
 		);
 
+		const end = this.kafkaMessageSummary
+			.labels({ type: message.type })
+			.startTimer();
+
 		if (message.type === SystemEventType.TWITCH_LINKED) {
 			const event = TwitchLinkedEvent.deserialize(message);
 
 			await client.join(event.twitchName);
-		}
 
-		if (message.type === SystemEventType.TWITCH_UNLINKED) {
+			end({ status: 200 });
+			const channelsGauge = this.metrics.register.getSingleMetric(
+				`${servicePrefix}_channels`,
+			);
+			if (channelsGauge) {
+				(channelsGauge as Gauge<string>).inc();
+			}
+		} else if (message.type === SystemEventType.TWITCH_UNLINKED) {
 			const event = TwitchLinkedEvent.deserialize(message);
 
 			await client.part(event.twitchName);
-		}
 
-		if (message.type === SystemEventType.TWITCH_MESSAGE_BROADCAST) {
+			end({ status: 200 });
+			const channelsGauge = this.metrics.register.getSingleMetric(
+				`${servicePrefix}_channels`,
+			);
+			if (channelsGauge) {
+				(channelsGauge as Gauge<string>).dec();
+			}
+		} else if (message.type === SystemEventType.TWITCH_MESSAGE_BROADCAST) {
 			const event = TwitchMessageBroadcastEvent.deserialize(message);
 
 			const userRepo = await this.postgres.getUserRepo();
@@ -76,6 +106,10 @@ export class BotCommandProcessor {
 				}
 				await sleep(2000);
 			}
+
+			end({ status: 200 });
+		} else {
+			end({ status: 404 });
 		}
 	}
 }