Quick Start -- Spark / MR / Hive Telemetry Deployment Guide¶

# Prerequisites: JDK 8, Maven 3.6+
git clone <repo-url> && cd spark-telemetry-listener

# Build Omnipackage (single JAR supporting Spark 2/3/4 + MR Agent/Collector + Hive Hook)
chmod +x build-omni.sh && ./build-omni.sh

# Build output location
ls spark/spark-telemetry-dist-omni/target/spark-telemetry-dist-omni-*.jar

# Build individual components (optional)
mvn clean package -pl hive/hive-telemetry-hook,hive/hive-telemetry-hook-dist -am -DskipTests      # Hive Hook
mvn clean package -pl flink/metrics-flink-consumer,flink/metrics-flink-consumer-dist -am -DskipTests  # Flink Consumer

# deploy/otel-collector/config.yaml
extensions:
  health_check:
    endpoint: 0.0.0.0:13133

receivers:
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317

exporters:
  kafka:
    topic: telemetry-metrics
    encoding: otlp_proto
    brokers:
      - kafka:9092

service:
  extensions: [health_check]
  pipelines:
    metrics:
      receivers: [otlp]
      exporters: [kafka]

# Docker deployment
docker run -d --name otel-collector --network host \
  -v $(pwd)/deploy/otel-collector/config.yaml:/etc/otelcol-contrib/config.yaml \
  otel/opentelemetry-collector-contrib:0.96.0 \
  --config=/etc/otelcol-contrib/config.yaml

# KRaft mode (no ZooKeeper required)
docker run -d --name kafka --network host \
  -e KAFKA_NODE_ID=1 \
  -e KAFKA_PROCESS_ROLES=broker,controller \
  -e KAFKA_LISTENERS=PLAINTEXT://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093 \
  -e KAFKA_CONTROLLER_QUORUM_VOTERS=1@localhost:9093 \
  -e KAFKA_CONTROLLER_LISTENER_NAMES=CONTROLLER \
  -e KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://$(hostname):9092 \
  apache/kafka:3.7.0

# Create topic
docker exec kafka /opt/kafka/bin/kafka-topics.sh --create \
  --topic telemetry-metrics --bootstrap-server localhost:9092 \
  --partitions 3 --replication-factor 1

# MySQL
docker run -d --name mysql --network host \
  -e MYSQL_ROOT_PASSWORD=root123 \
  -e MYSQL_DATABASE=metrics_db \
  mysql:8.0

# ClickHouse (optional)
docker run -d --name clickhouse --network host \
  clickhouse/clickhouse-server:23.8

# Create configuration file flink-consumer.conf
# Refer to conf/flink/flink-consumer-mysql.conf or flink-consumer-clickhouse.conf

# Standalone run (no Flink cluster needed, uses Flink LocalEnvironment)
java -jar metrics-flink-consumer-dist-1.0.0-SNAPSHOT.jar /path/to/flink-consumer.conf

# Submit to Flink cluster (recommended for production)
/opt/flink-1.18.0/bin/flink run -c x.mg.metrics.flink.Main \
  -m localhost:8081 \
  /path/to/metrics-flink-consumer-dist-1.0.0-SNAPSHOT.jar /path/to/flink-consumer.conf

# Background submission (survives SSH disconnection)
nohup /opt/flink-1.18.0/bin/flink run -c x.mg.metrics.flink.Main \
  -m localhost:8081 \
  /path/to/metrics-flink-consumer-dist-1.0.0-SNAPSHOT.jar /path/to/flink-consumer.conf \
  > /tmp/flink-submit.log 2>&1 &

# Check cluster job status
curl -s http://localhost:8081/jobs | python3 -c \
  'import json,sys; [print(j["id"],j["status"]) for j in json.load(sys.stdin)["jobs"]]'

# Cancel a job
/opt/flink-1.18.0/bin/flink cancel <job-id>

docker run -d --name grafana --network host \
  -e GF_SECURITY_ADMIN_PASSWORD=admin123 \
  grafana/grafana:latest

scp spark-telemetry-dist-omni-*.jar node:/opt/spark-telemetry-plugin.jar

spark-submit --master yarn \
  --jars /opt/spark-telemetry-plugin.jar \
  --conf spark.plugins=x.mg.metrics.sparktelemetry.adapter.SparkTelemetryPlugin \
  --conf spark.telemetry.otel.exporter.endpoint=http://otel-collector:4317 \
  --conf spark.telemetry.otel.service.name=my-spark-app \
  your-app.jar

spark-submit --master yarn \
  --jars /opt/spark-telemetry-plugin.jar \
  --conf spark.extraListeners=x.mg.metrics.sparktelemetry.adapter.SparkTelemetryListener \
  --conf spark.telemetry.otel.exporter.endpoint=http://otel-collector:4317 \
  --conf spark.telemetry.otel.service.name=my-spark-app \
  your-app.jar

# Required
spark.telemetry.otel.exporter.endpoint=http://otel-collector:4317

# Recommended configuration
spark.telemetry.otel.service.name=my-spark-app          # Service name, used for Grafana filtering
spark.telemetry.otel.export.interval.ms=10000            # Export interval (default 10s)
spark.telemetry.config.path=/etc/telemetry.conf          # Config file path

# Metric switches (default values below)
spark.telemetry.metrics.task.execution=true              # task CPU/GC/memory/duration
spark.telemetry.metrics.task.shuffle-extended=true       # detailed shuffle metrics
spark.telemetry.metrics.task.info=true                   # task host/locality info
spark.telemetry.metrics.stage.detailed=true              # stage-level metrics
spark.telemetry.metrics.job.lifecycle=true               # job lifecycle events
spark.telemetry.metrics.sql.query-execution=true         # SQL execution metrics (join/shuffle bytes/query text)
spark.telemetry.sql.max-length=4096                      # SQL text max truncation length (characters)

# 1. Copy JAR to Hadoop node
scp spark-telemetry-dist-omni-*.jar node:/opt/hadoop/lib/mr-telemetry-agent.jar

# 2. Configure mapred-site.xml

<!-- mapred-site.xml -->
<property>
  <name>mapreduce.framework.name</name>
  <value>yarn</value>
</property>
<property>
  <name>mapreduce.map.java.opts</name>
  <value>-javaagent:/opt/hadoop/lib/mr-telemetry-agent.jar -Dmr.telemetry.agent.otel.exporter.endpoint=http://otel-collector:4317</value>
</property>
<property>
  <name>mapreduce.reduce.java.opts</name>
  <value>-javaagent:/opt/hadoop/lib/mr-telemetry-agent.jar -Dmr.telemetry.agent.otel.exporter.endpoint=http://otel-collector:4317</value>
</property>

# 1. Create configuration file mr-collector.conf
# Refer to conf/examples/mr-collector.conf.example or conf/spark35-hadoop3-hive2.3.9/mr-collector.conf

# mr-collector.conf
mr-telemetry {
  history-server {
    url = "http://hadoop-historyserver:19888"
    poll.interval.secs = 30
  }
  otel {
    exporter.endpoint = "http://otel-collector:4317"
    service.name = "mr-telemetry-collector"
    export.interval.ms = 10000
  }
  state {
    file = "/tmp/mr-telemetry-state.json"   # Resume from checkpoint, no duplicate collection after restart
  }
  collection {
    job.counters = true      # job-level: HDFS IO, CPU, GC, maps/reduces count
    task.counters = false    # task-level (large data volume, enable on demand)
    job.details = true       # job name, user, status
  }
}

# 2. Run (using Omnipackage)
java -jar spark-telemetry-dist-omni-*.jar --mr-collector /path/to/mr-collector.conf

# 3. For production, use systemd or nohup for background execution
nohup java -jar spark-telemetry-dist-omni-*.jar --mr-collector /etc/mr-collector.conf &

# 1. Place the Omnipackage JAR in HiveServer2 auxlib (auto-loaded, no need for hive.aux.jars.path)
cp spark-telemetry-dist-omni-*.jar $HIVE_HOME/auxlib/

<!-- hive-site.xml -->
<property>
  <name>hive.exec.post.hooks</name>
  <value>x.mg.metrics.hivetelemetry.HiveTelemetryHook</value>
</property>
<property>
  <name>hive.telemetry.otel.exporter.endpoint</name>
  <value>http://otel-collector:4317</value>
</property>
<property>
  <name>hive.telemetry.sql.max-length</name>
  <value>4096</value>
</property>

docker logs otel-collector --tail=50 | grep -E "spark\.|mr\.|hive\."

docker exec kafka /opt/kafka/bin/kafka-dump-log.sh \
  --files /tmp/kafka-logs/telemetry-metrics-0/00000000000000000000.log

-- Spark metrics
SELECT COUNT(*) FROM task_metrics;
SELECT COUNT(*) FROM sql_query_metrics;

-- MR metrics
SELECT COUNT(*) FROM mr_job_metrics;    -- MR Collector
SELECT COUNT(*) FROM mr_task_metrics;   -- MR Agent

-- Hive metrics
SELECT COUNT(*) FROM hive_query_metrics;
SELECT COUNT(*) FROM hive_table_io_metrics;

-- Unified wide table
SELECT engine, event_type, COUNT(*) FROM metric_events GROUP BY engine, event_type;

# Build diagnostic tool
mvn clean package -pl diagnostic/diagnostic-core -am -DskipTests

# Run (interactive CLI, JLine terminal)
java -jar diagnostic/diagnostic-core/target/diagnostic-core-1.0.0-SNAPSHOT.jar

# Specify configuration file
java -jar diagnostic/diagnostic-core/target/diagnostic-core-1.0.0-SNAPSHOT.jar \
  --config /path/to/diagnostic.conf

URL: mysql:3306
Database: metrics_db
User: metrics / Password: metrics