Spark JMX Metrics

Every 10 seconds, the collectd service gathers the following Spark JMX metrics on each node in the cluster.

For detailed information about Spark metrics, see the Apache Spark documentation.

mapr.spark.driver_block_manager_disk_space_used_mb
mapr.spark.driver_block_manager_memory_max_mem_mbb
mapr.spark.driver_block_manager_memory_max_off_heap_mem_mb
mapr.spark.driver_block_manager_memory_max_on_heap_mem_mb
mapr.spark.driver_block_manager_memory_mem_used_mb
mapr.spark.driver_block_manager_memory_off_heap_mem_used_mb
mapr.spark.driver_block_manager_memory_on_heap_mem_used_mb
mapr.spark.driver_block_manager_memory_remaining_mem_mb
mapr.spark.driver_block_manager_memory_remaining_off_heap_mem_mb
mapr.spark.driver_block_manager_memory_remaining_on_heap_mem_mb
mapr.spark.driver_dag_scheduler_job_active_jobs
mapr.spark.driver_dag_scheduler_job_all_jobs
mapr.spark.driver_dag_scheduler_stage_failed_stages
mapr.spark.driver_dag_scheduler_stage_running_stages
mapr.spark.driver_dag_scheduler_stage_waiting_stages
mapr.spark.driver_live_listener_bus_queue_app_status_size
mapr.spark.driver_live_listener_bus_queue_event_log_size
mapr.spark.driver_live_listener_bus_queue_executor_management_size
mapr.spark.driver_hive_external_catalog_file_cache_hits
mapr.spark.driver_hive_external_catalog_files_discovered
mapr.spark.driver_hive_external_catalog_hive_client_calls
mapr.spark.driver_hive_external_catalog_parallel_listing_job_count
mapr.spark.driver_hive_external_catalog_partitions_fetched
mapr.spark.driver_live_listener_bus_num_events_posted
mapr.spark.driver_live_listener_bus_queue_app_status_num_dropped_events
mapr.spark.driver_live_listener_bus_queue_event_log_num_dropped_events
mapr.spark.driver_live_listener_bus_queue_executor_management_num_dropped_events
mapr.spark.driver_code_generator_compilation_time_count
mapr.spark.driver_code_generator_compilation_time_max
mapr.spark.driver_code_generator_compilation_time_mean
mapr.spark.driver_code_generator_compilation_time_min
mapr.spark.driver_code_generator_compilation_time_p50
mapr.spark.driver_code_generator_compilation_time_p75
mapr.spark.driver_code_generator_compilation_time_p95
mapr.spark.driver_code_generator_compilation_time_p98
mapr.spark.driver_code_generator_compilation_time_p99
mapr.spark.driver_code_generator_compilation_time_p999
mapr.spark.driver_code_generator_compilation_time_stddev
mapr.spark.driver_code_generator_generated_class_size_count
mapr.spark.driver_code_generator_generated_class_size_max
mapr.spark.driver_code_generator_generated_class_size_mean
mapr.spark.driver_code_generator_generated_class_size_min
mapr.spark.driver_code_generator_generated_class_size_p50
mapr.spark.driver_code_generator_generated_class_size_p75
mapr.spark.driver_code_generator_generated_class_size_p95
mapr.spark.driver_code_generator_generated_class_size_p98
mapr.spark.driver_code_generator_generated_class_size_p99
mapr.spark.driver_code_generator_generated_class_size_p999
mapr.spark.driver_code_generator_generated_class_size_stddev
mapr.spark.driver_code_generator_generated_method_size_count
mapr.spark.driver_code_generator_generated_method_size_max
mapr.spark.driver_code_generator_generated_method_size_mean
mapr.spark.driver_code_generator_generated_method_size_min
mapr.spark.driver_code_generator_generated_method_size_p50
mapr.spark.driver_code_generator_generated_method_size_p75
mapr.spark.driver_code_generator_generated_method_size_p95
mapr.spark.driver_code_generator_generated_method_size_p98
mapr.spark.driver_code_generator_generated_method_size_p99
mapr.spark.driver_code_generator_generated_method_size_p999
mapr.spark.driver_code_generator_generated_method_size_stddev
mapr.spark.driver_code_generator_source_code_size_count
mapr.spark.driver_code_generator_source_code_size_max
mapr.spark.driver_code_generator_source_code_size_mean
mapr.spark.driver_code_generator_source_code_size_min
mapr.spark.driver_code_generator_source_code_size_p50
mapr.spark.driver_code_generator_source_code_size_p75
mapr.spark.driver_code_generator_source_code_size_p95
mapr.spark.driver_code_generator_source_code_size_p98
mapr.spark.driver_code_generator_source_code_size_p99
mapr.spark.driver_code_generator_source_code_size_p999
mapr.spark.driver_code_generator_source_code_size_stddev
mapr.spark.driver_dag_scheduler_message_processing_time_count
mapr.spark.driver_dag_scheduler_message_processing_time_max
mapr.spark.driver_dag_scheduler_message_processing_time_mean
mapr.spark.driver_dag_scheduler_message_processing_time_min
mapr.spark.driver_dag_scheduler_message_processing_time_p50
mapr.spark.driver_dag_scheduler_message_processing_time_p75
mapr.spark.driver_dag_scheduler_message_processing_time_p95
mapr.spark.driver_dag_scheduler_message_processing_time_p98
mapr.spark.driver_dag_scheduler_message_processing_time_p99
mapr.spark.driver_dag_scheduler_message_processing_time_p999
mapr.spark.driver_dag_scheduler_message_processing_time_stddev
mapr.spark.driver_dag_scheduler_message_processing_time_m15_rate
mapr.spark.driver_dag_scheduler_message_processing_time_m1_rate
mapr.spark.driver_dag_scheduler_message_processing_time_m5_rate
mapr.spark.driver_dag_scheduler_message_processing_time_mean_rate
mapr.spark.driver_live_listener_bus_listener_processing_time_heartbeat_receiver_count
mapr.spark.driver_live_listener_bus_listener_processing_time_heartbeat_receiver_max
mapr.spark.driver_live_listener_bus_listener_processing_time_heartbeat_receiver_mean
mapr.spark.driver_live_listener_bus_listener_processing_time_heartbeat_receiver_min
mapr.spark.driver_live_listener_bus_listener_processing_time_heartbeat_receiver_p50
mapr.spark.driver_live_listener_bus_listener_processing_time_heartbeat_receiver_p75
mapr.spark.driver_live_listener_bus_listener_processing_time_heartbeat_receiver_p95
mapr.spark.driver_live_listener_bus_listener_processing_time_heartbeat_receiver_p98
mapr.spark.driver_live_listener_bus_listener_processing_time_heartbeat_receiver_p99
mapr.spark.driver_live_listener_bus_listener_processing_time_heartbeat_receiver_p999
mapr.spark.driver_live_listener_bus_listener_processing_time_heartbeat_receiver_stddev
mapr.spark.driver_live_listener_bus_listener_processing_time_heartbeat_receiver_m15_rate
mapr.spark.driver_live_listener_bus_listener_processing_time_heartbeat_receiver_m1_rate
mapr.spark.driver_live_listener_bus_listener_processing_time_heartbeat_receiver_m5_rate
mapr.spark.driver_live_listener_bus_listener_processing_time_heartbeat_receiver_mean_rate
mapr.spark.driver_live_listener_bus_listener_processing_time_event_logging_listener_count
mapr.spark.driver_live_listener_bus_listener_processing_time_event_logging_listener_max
mapr.spark.driver_live_listener_bus_listener_processing_time_event_logging_listener_mean
mapr.spark.driver_live_listener_bus_listener_processing_time_event_logging_listener_min
mapr.spark.driver_live_listener_bus_listener_processing_time_event_logging_listener_p50
mapr.spark.driver_live_listener_bus_listener_processing_time_event_logging_listener_p75
mapr.spark.driver_live_listener_bus_listener_processing_time_event_logging_listener_p95
mapr.spark.driver_live_listener_bus_listener_processing_time_event_logging_listener_p98
mapr.spark.driver_live_listener_bus_listener_processing_time_event_logging_listener_p99
mapr.spark.driver_live_listener_bus_listener_processing_time_event_logging_listener_p999
mapr.spark.driver_live_listener_bus_listener_processing_time_event_logging_listener_stddev
mapr.spark.driver_live_listener_bus_listener_processing_time_event_logging_listener_m15_rate
mapr.spark.driver_live_listener_bus_listener_processing_time_event_logging_listener_m1_rate
mapr.spark.driver_live_listener_bus_listener_processing_time_event_logging_listener_m5_rate
mapr.spark.driver_live_listener_bus_listener_processing_time_event_logging_listener_mean_rate
mapr.spark.driver_live_listener_bus_listener_processing_time_app_status_listener_count
mapr.spark.driver_live_listener_bus_listener_processing_time_app_status_listener_max
mapr.spark.driver_live_listener_bus_listener_processing_time_app_status_listener_mean
mapr.spark.driver_live_listener_bus_listener_processing_time_app_status_listener_min
mapr.spark.driver_live_listener_bus_listener_processing_time_app_status_listener_p50
mapr.spark.driver_live_listener_bus_listener_processing_time_app_status_listener_p75
mapr.spark.driver_live_listener_bus_listener_processing_time_app_status_listener_p95
mapr.spark.driver_live_listener_bus_listener_processing_time_app_status_listener_p98
mapr.spark.driver_live_listener_bus_listener_processing_time_app_status_listener_p99
mapr.spark.driver_live_listener_bus_listener_processing_time_app_status_listener_p999
mapr.spark.driver_live_listener_bus_listener_processing_time_app_status_listener_stddev
mapr.spark.driver_live_listener_bus_listener_processing_time_app_status_listener_m15_rate
mapr.spark.driver_live_listener_bus_listener_processing_time_app_status_listener_m1_rate
mapr.spark.driver_live_listener_bus_listener_processing_time_app_status_listener_m5_rate
mapr.spark.driver_live_listener_bus_listener_processing_time_app_status_listener_mean_rate
mapr.spark.driver_live_listener_bus_queue_app_status_listener_processing_time_count
mapr.spark.driver_live_listener_bus_queue_app_status_listener_processing_time_max
mapr.spark.driver_live_listener_bus_queue_app_status_listener_processing_time_mean
mapr.spark.driver_live_listener_bus_queue_app_status_listener_processing_time_min
mapr.spark.driver_live_listener_bus_queue_app_status_listener_processing_time_p50
mapr.spark.driver_live_listener_bus_queue_app_status_listener_processing_time_p75
mapr.spark.driver_live_listener_bus_queue_app_status_listener_processing_time_p95
mapr.spark.driver_live_listener_bus_queue_app_status_listener_processing_time_p98
mapr.spark.driver_live_listener_bus_queue_app_status_listener_processing_time_p99
mapr.spark.driver_live_listener_bus_queue_app_status_listener_processing_time_p999
mapr.spark.driver_live_listener_bus_queue_app_status_listener_processing_time_stddev
mapr.spark.driver_live_listener_bus_queue_app_status_listener_processing_time_m15_rat
mapr.spark.driver_live_listener_bus_queue_app_status_listener_processing_time_m1_rate
mapr.spark.driver_live_listener_bus_queue_app_status_listener_processing_time_m5_rate
mapr.spark.driver_live_listener_bus_queue_app_status_listener_processing_time_mean_rate
mapr.spark.driver_live_listener_bus_queue_event_log_listener_processing_time_count
mapr.spark.driver_live_listener_bus_queue_event_log_listener_processing_time_max
mapr.spark.driver_live_listener_bus_queue_event_log_listener_processing_time_mean
mapr.spark.driver_live_listener_bus_queue_event_log_listener_processing_time_min
mapr.spark.driver_live_listener_bus_queue_event_log_listener_processing_time_p50
mapr.spark.driver_live_listener_bus_queue_event_log_listener_processing_time_p75
mapr.spark.driver_live_listener_bus_queue_event_log_listener_processing_time_p95
mapr.spark.driver_live_listener_bus_queue_event_log_listener_processing_time_p98
mapr.spark.driver_live_listener_bus_queue_event_log_listener_processing_time_p99
mapr.spark.driver_live_listener_bus_queue_event_log_listener_processing_time_p999
mapr.spark.driver_live_listener_bus_queue_event_log_listener_processing_time_stddev
mapr.spark.driver_live_listener_bus_queue_event_log_listener_processing_time_m15_rate
mapr.spark.driver_live_listener_bus_queue_event_log_listener_processing_time_m1_rate
mapr.spark.driver_live_listener_bus_queue_event_log_listener_processing_time_m5_rate
mapr.spark.driver_live_listener_bus_queue_event_log_listener_processing_time_mean_rate
mapr.spark.driver_live_listener_bus_queue_executor_management_listener_processing_time_count
mapr.spark.driver_live_listener_bus_queue_executor_management_listener_processing_time_max
mapr.spark.driver_live_listener_bus_queue_executor_management_listener_processing_time_mean
mapr.spark.driver_live_listener_bus_queue_executor_management_listener_processing_time_min
mapr.spark.driver_live_listener_bus_queue_executor_management_listener_processing_time_p50
mapr.spark.driver_live_listener_bus_queue_executor_management_listener_processing_time_p75
mapr.spark.driver_live_listener_bus_queue_executor_management_listener_processing_time_p95
mapr.spark.driver_live_listener_bus_queue_executor_management_listener_processing_time_p98
mapr.spark.driver_live_listener_bus_queue_executor_management_listener_processing_time_p99
mapr.spark.driver_live_listener_bus_queue_executor_management_listener_processing_time_p990
mapr.spark.driver_live_listener_bus_queue_executor_management_listener_processing_time_stddev
mapr.spark.driver_live_listener_bus_queue_executor_management_listener_processing_time_m15_rate
mapr.spark.driver_live_listener_bus_queue_executor_management_listener_processing_time_m1_rate
mapr.spark.driver_live_listener_bus_queue_executor_management_listener_processing_time_m5_rate
mapr.spark.driver_live_listener_bus_queue_executor_management_listener_processing_time_mean_rate
mapr.spark.driver_jvm_ps_mark_sweep_count
mapr.spark.driver_jvm_ps_mark_sweep_time
mapr.spark.driver_jvm_ps_scavenge_count
mapr.spark.driver_jvm_ps_scavenge_time
mapr.spark.driver_jvm_direct_capacity
mapr.spark.driver_jvm_direct_count
mapr.spark.driver_jvm_direct_used
mapr.spark.driver_jvm_direct_heap_committed
mapr.spark.driver_jvm_direct_heap_init
mapr.spark.driver_jvm_direct_heap_max
mapr.spark.driver_jvm_direct_heap_usage
mapr.spark.driver_jvm_direct_heap_used
mapr.spark.driver_jvm_direct_mapped_capacity
mapr.spark.driver_jvm_direct_mapped_count
mapr.spark.driver_jvm_direct_mapped_used
mapr.spark.driver_jvm_direct_non_heap_committed
mapr.spark.driver_jvm_direct_non_heap_init
mapr.spark.driver_jvm_direct_non_heap_max
mapr.spark.driver_jvm_direct_non_heap_usage
mapr.spark.driver_jvm_direct_non_heap_used
mapr.spark.driver_jvm_pools_code_cache_committed
mapr.spark.driver_jvm_pools_code_cache_init
mapr.spark.driver_jvm_pools_code_cache_max
mapr.spark.driver_jvm_pools_code_cache_usage
mapr.spark.driver_jvm_pools_code_cache_used
mapr.spark.driver_jvm_pools_compressed_class_space_committed
mapr.spark.driver_jvm_pools_compressed_class_space_init
mapr.spark.driver_jvm_pools_compressed_class_space_max
mapr.spark.driver_jvm_pools_compressed_class_space_usage
mapr.spark.driver_jvm_pools_compressed_class_space_used
mapr.spark.driver_jvm_pools_metaspace_committed
mapr.spark.driver_jvm_pools_metaspace_init
mapr.spark.driver_jvm_pools_metaspace_max
mapr.spark.driver_jvm_pools_metaspace_usage
mapr.spark.driver_jvm_pools_metaspace_used
mapr.spark.driver_jvm_pools_ps_eden_space_committed
mapr.spark.driver_jvm_pools_ps_eden_space_init
mapr.spark.driver_jvm_pools_ps_eden_space_max
mapr.spark.driver_jvm_pools_ps_eden_space_usage
mapr.spark.driver_jvm_pools_ps_eden_space_used
mapr.spark.driver_jvm_pools_ps_old_gen_committed
mapr.spark.driver_jvm_pools_ps_old_gen_init
mapr.spark.driver_jvm_pools_ps_old_gen_max
mapr.spark.driver_jvm_pools_ps_old_gen_usage
mapr.spark.driver_jvm_pools_ps_old_gen_used
mapr.spark.driver_jvm_pools_ps_survivor_space_committed
mapr.spark.driver_jvm_pools_ps_survivor_space_init
mapr.spark.driver_jvm_pools_ps_survivor_space_max
mapr.spark.driver_jvm_pools_ps_survivor_space_usage
mapr.spark.driver_jvm_pools_ps_survivor_space_used
mapr.spark.driver_jvm_total_committed
mapr.spark.driver_jvm_total_init
mapr.spark.driver_jvm_total_max
mapr.spark.driver_jvm_total_used