From fefaaf1a7ae608aa7213b26120557dd13788c4d6 Mon Sep 17 00:00:00 2001 From: Leonardo Menezes Date: Fri, 6 Oct 2017 11:01:51 +0200 Subject: [PATCH] optimised overview data api --- .../ClusterOverviewController.scala | 39 +- app/models/ShardStats.scala | 14 +- app/models/commons/NodeRoles.scala | 4 + app/models/overview/ClosedIndices.scala | 18 - app/models/overview/ClusterOverview.scala | 64 -- app/models/overview/Index.scala | 49 - app/models/overview/Node.scala | 67 -- app/services/overview/ClusterOverview.scala | 87 ++ app/services/overview/DataService.scala | 62 ++ app/services/overview/Index.scala | 44 + app/services/overview/Node.scala | 47 + app/util/DataSize.scala | 20 + public/js/app.js | 56 +- public/overview.html | 118 ++- public/stats.html | 6 +- src/app/components/overview/controller.js | 36 +- src/app/shared/index_filter.js | 12 +- src/app/shared/node_filter.js | 8 +- test/controllers/OverviewControllerSpec.scala | 8 +- .../overview/ClusterInitializingShards.scala | 721 --------------- .../overview/ClusterRelocatingShards.scala | 862 ------------------ test/models/overview/ClusterStub.scala | 25 - test/models/overview/ClusterWithData.scala | 715 --------------- test/models/overview/ClusterWithoutData.scala | 462 ---------- test/models/overview/NodeSpec.scala | 88 -- test/models/overview/NodesInfo.scala | 95 -- .../overview/ClusterDisabledAllocation.scala | 4 +- .../overview/ClusterInitializingShards.scala | 167 ++++ .../overview/ClusterOverviewSpec.scala | 70 +- .../overview/ClusterRelocatingShards.scala | 147 +++ test/services/overview/ClusterStub.scala | 21 + test/services/overview/ClusterWithData.scala | 134 +++ .../overview/ClusterWithoutData.scala | 81 ++ test/services/overview/NodeSpec.scala | 82 ++ .../overview/NodeStats.scala | 2 +- test/services/overview/NodesInfo.scala | 48 + tests/common/index_filter.js | 2 +- tests/controllers/overview.tests.js | 39 +- 38 files changed, 1112 insertions(+), 3412 deletions(-) delete mode 100644 app/models/overview/ClosedIndices.scala delete mode 100644 app/models/overview/ClusterOverview.scala delete mode 100644 app/models/overview/Index.scala delete mode 100644 app/models/overview/Node.scala create mode 100644 app/services/overview/ClusterOverview.scala create mode 100644 app/services/overview/DataService.scala create mode 100644 app/services/overview/Index.scala create mode 100644 app/services/overview/Node.scala create mode 100644 app/util/DataSize.scala delete mode 100644 test/models/overview/ClusterInitializingShards.scala delete mode 100644 test/models/overview/ClusterRelocatingShards.scala delete mode 100644 test/models/overview/ClusterStub.scala delete mode 100644 test/models/overview/ClusterWithData.scala delete mode 100644 test/models/overview/ClusterWithoutData.scala delete mode 100644 test/models/overview/NodeSpec.scala delete mode 100644 test/models/overview/NodesInfo.scala rename test/{models => services}/overview/ClusterDisabledAllocation.scala (84%) create mode 100644 test/services/overview/ClusterInitializingShards.scala rename test/{models => services}/overview/ClusterOverviewSpec.scala (51%) create mode 100644 test/services/overview/ClusterRelocatingShards.scala create mode 100644 test/services/overview/ClusterStub.scala create mode 100644 test/services/overview/ClusterWithData.scala create mode 100644 test/services/overview/ClusterWithoutData.scala create mode 100644 test/services/overview/NodeSpec.scala rename test/{models => services}/overview/NodeStats.scala (99%) create mode 100644 test/services/overview/NodesInfo.scala diff --git a/app/controllers/ClusterOverviewController.scala b/app/controllers/ClusterOverviewController.scala index 5d253db..b7c072b 100644 --- a/app/controllers/ClusterOverviewController.scala +++ b/app/controllers/ClusterOverviewController.scala @@ -4,45 +4,20 @@ import javax.inject.Inject import controllers.auth.AuthenticationModule import elastic.{ElasticClient, Error} -import models.overview.ClusterOverview import models.{CerebroResponse, Hosts, ShardStats} +import services.overview.DataService import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.Future class ClusterOverviewController @Inject()(val authentication: AuthenticationModule, val hosts: Hosts, - client: ElasticClient) extends BaseController { + client: ElasticClient, + data: DataService) extends BaseController { def index = process { request => - Future.sequence( - Seq( - client.clusterState(request.target), - client.nodesStats(Seq("jvm","fs","os","process"), request.target), - client.indicesStats(request.target), - client.clusterSettings(request.target), - client.aliases(request.target), - client.clusterHealth(request.target), - client.nodes(Seq("os","jvm"), request.target), - client.main(request.target) - ) - ).map { responses => - val failed = responses.find(_.isInstanceOf[Error]) - failed match { - case Some(f) => CerebroResponse(f.status, f.body) - case None => - val overview = ClusterOverview( - responses(0).body, - responses(1).body, - responses(2).body, - responses(3).body, - responses(4).body, - responses(5).body, - responses(6).body, - responses(7).body - ) - CerebroResponse(200, overview) - } + data.getOverviewData(request.target).map { overview => + CerebroResponse(200, overview) } } @@ -102,7 +77,7 @@ class ClusterOverviewController @Inject()(val authentication: AuthenticationModu def getShardStats = process { request => val index = request.get("index") - val shard = request.getInt("shard") + val shard = request.get("shard").toInt // TODO ES return as Int? val node = request.get("node") Future.sequence( Seq( @@ -124,7 +99,7 @@ class ClusterOverviewController @Inject()(val authentication: AuthenticationModu def relocateShard = process { request => val index = request.get("index") - val shard = request.getInt("shard") + val shard = request.get("shard").toInt // TODO ES return as Int? val from = request.get("from") val to = request.get("to") val server = request.target diff --git a/app/models/ShardStats.scala b/app/models/ShardStats.scala index 85c3084..e16331b 100644 --- a/app/models/ShardStats.scala +++ b/app/models/ShardStats.scala @@ -9,20 +9,20 @@ object ShardStats { shardStats.getOrElse(getShardRecovery(index, node, shard, recovery).getOrElse(JsNull)) } + private def matchesNode(node: String, shardNode: String): Boolean = { + shardNode.equals(node) || shardNode.startsWith(node) + } + private def getShardStats(index: String, node: String, shard: Int, stats: JsValue): Option[JsValue] = (stats \ "indices" \ index \ "shards" \ shard.toString).asOpt[JsArray] match { - case Some(JsArray(shards)) => shards.collectFirst { - case s if (s \ "routing" \ "node").as[String].equals(node) => s - } + case Some(JsArray(shards)) => shards.find(s => matchesNode(node, (s \ "routing" \ "node").as[String])) case _ => None } - private def getShardRecovery(index: String, node: String, shard: Int, recovery: JsValue): Option[JsValue] = (recovery \ index \ "shards").asOpt[JsArray] match { - case Some(JsArray(recoveries)) => recoveries.collectFirst { - case r if (r \ "target" \ "id").as[String].equals(node) && (r \ "id").as[Int].equals(shard) => r - } + case Some(JsArray(recoveries)) => + recoveries.find(r => matchesNode(node, (r \ "target" \ "id").as[String]) && (r \ "id").as[Int] == shard) case _ => None } diff --git a/app/models/commons/NodeRoles.scala b/app/models/commons/NodeRoles.scala index 33aef89..1161933 100644 --- a/app/models/commons/NodeRoles.scala +++ b/app/models/commons/NodeRoles.scala @@ -35,4 +35,8 @@ object NodeRoles { ) } } + + def apply(roles: String): NodeRoles = + NodeRoles(roles.contains("m"), roles.contains("d"), roles.contains("i")) + } diff --git a/app/models/overview/ClosedIndices.scala b/app/models/overview/ClosedIndices.scala deleted file mode 100644 index 16a49b9..0000000 --- a/app/models/overview/ClosedIndices.scala +++ /dev/null @@ -1,18 +0,0 @@ -package models.overview - -import play.api.libs.json._ - -object ClosedIndices { - - def apply(clusterState: JsValue) = { - val blocks = (clusterState \ "blocks" \ "indices").asOpt[JsObject].getOrElse(Json.obj()) - blocks.keys.collect { - case index if (blocks \ index \ "4").asOpt[JsObject].isDefined => - Json.obj( - "name" -> JsString(index), - "closed" -> JsBoolean(true), - "special" -> JsBoolean(index.startsWith(".")) - ) - } - } -} diff --git a/app/models/overview/ClusterOverview.scala b/app/models/overview/ClusterOverview.scala deleted file mode 100644 index e53e930..0000000 --- a/app/models/overview/ClusterOverview.scala +++ /dev/null @@ -1,64 +0,0 @@ -package models.overview - -import play.api.libs.json._ - -object ClusterOverview { - - def apply(clusterState: JsValue, nodesStats: JsValue, indicesStats: JsValue, - clusterSettings: JsValue, aliases: JsValue, clusterHealth: JsValue, - nodesInfo: JsValue, main: JsValue): JsValue = { - - val indices = buildIndices(clusterState, indicesStats, aliases) - - val masterNodeId = (clusterState \ "master_node").as[String] - - val persistentAllocation = (clusterSettings \ "persistent" \ "cluster" \ "routing" \ "allocation" \ "enable").asOpt[String].getOrElse("all") - val transientAllocation = (clusterSettings \ "transient" \ "cluster" \ "routing" \ "allocation" \ "enable").asOpt[String] - val shardAllocation = transientAllocation.getOrElse(persistentAllocation).equals("all") - - JsObject(Seq( - // clusterHealth - "cluster_name" -> (clusterHealth \ "cluster_name").as[JsString], - "status" -> (clusterHealth \ "status").as[JsString], - "number_of_nodes" -> (clusterHealth \ "number_of_nodes").as[JsNumber], - "active_primary_shards" -> (clusterHealth \ "active_primary_shards").as[JsNumber], - "active_shards" -> (clusterHealth \ "active_shards").as[JsNumber], - "relocating_shards" -> (clusterHealth \ "relocating_shards").as[JsNumber], - "initializing_shards" -> (clusterHealth \ "initializing_shards").as[JsNumber], - "unassigned_shards" -> (clusterHealth \ "unassigned_shards").as[JsNumber], - // indicesStats - "docs_count" -> (indicesStats \ "_all" \ "primaries" \ "docs" \ "count").asOpt[JsNumber].getOrElse(JsNumber(0)), - "size_in_bytes" -> (indicesStats \ "_all" \ "total" \ "store" \ "size_in_bytes").asOpt[JsNumber].getOrElse(JsNumber(0)), - "total_indices" -> JsNumber(indices.size), - "closed_indices" -> JsNumber(indices.count { idx => (idx \ "closed").as[Boolean] }), - "special_indices" -> JsNumber(indices.count { idx => (idx \ "special").as[Boolean] }), - "indices" -> JsArray(indices), - "nodes" -> buildNodes(masterNodeId, nodesInfo, nodesStats), - "shard_allocation" -> JsBoolean(shardAllocation) - )) - } - - def buildNodes(masterNodeId: String, nodesInfo: JsValue, nodesStats: JsValue): JsArray = - JsArray( - (nodesInfo \ "nodes").as[JsObject].value.map { - case (id, info) => - val stats = (nodesStats \ "nodes" \ id).as[JsObject] - Node(id, info, stats, masterNodeId) - }.toSeq - ) - - def buildIndices(clusterState: JsValue, indicesStats: JsValue, aliases: JsValue): Seq[JsValue] = { - val routingTable = (clusterState \ "routing_table" \ "indices").as[JsObject] - val routingNodes = (clusterState \ "routing_nodes" \ "nodes").as[JsObject] - val openIndices = routingTable.value.map { case (index, shards) => - val indexStats = (indicesStats \ "indices" \ index).asOpt[JsObject].getOrElse(Json.obj()) - val indexAliases = (aliases \ index \ "aliases").asOpt[JsObject].getOrElse(Json.obj()) // 1.4 < does not return aliases obj - - Index(index, indexStats, shards, indexAliases, routingNodes) - }.toSeq - - val closedIndices = ClosedIndices(clusterState) - openIndices ++ closedIndices - } - -} diff --git a/app/models/overview/Index.scala b/app/models/overview/Index.scala deleted file mode 100644 index d44a9b7..0000000 --- a/app/models/overview/Index.scala +++ /dev/null @@ -1,49 +0,0 @@ -package models.overview - -import play.api.libs.json._ - -object Index { - - def apply(name: String, stats: JsValue, shards: JsValue, aliases: JsObject, routingNodes: JsValue): JsValue = { - - val unassignedShards = (shards \ "shards").as[JsObject].values.flatMap { - case JsArray(shards) => - shards.filter { shard => - (shard \ "node").asOpt[String].isEmpty - } - case _ => Nil - } - - val shardsAllocation = routingNodes.as[JsObject].value.mapValues { - case JsArray(shards) => JsArray(shards.filter { shard => (shard \ "index").as[String].equals(name) }) - case _ => JsArray() - }.toSeq ++ Seq("unassigned" -> JsArray(unassignedShards.toSeq)) - - val numShards = (shards \ "shards").as[JsObject].keys.size - val numReplicas = (shards \ "shards" \ "0").as[JsArray].value.size - 1 - - val special = name.startsWith(".") - - JsObject(Seq( - "name" -> JsString(name), - "closed" -> JsBoolean(false), - "special" -> JsBoolean(special), - "unhealthy" -> JsBoolean(unhealthyIndex(shardsAllocation)), - "doc_count" -> (stats \ "primaries" \ "docs" \ "count").asOpt[JsNumber].getOrElse(JsNumber(0)), - "deleted_docs" -> (stats \ "primaries" \ "docs" \ "deleted").asOpt[JsNumber].getOrElse(JsNumber(0)), - "size_in_bytes" -> (stats \ "primaries" \ "store" \ "size_in_bytes").asOpt[JsNumber].getOrElse(JsNumber(0)), - "total_size_in_bytes" -> (stats \ "total" \ "store" \ "size_in_bytes").asOpt[JsNumber].getOrElse(JsNumber(0)), - "aliases" -> JsArray(aliases.keys.map(JsString(_)).toSeq), // 1.4 < does not return aliases obj - "num_shards" -> JsNumber(numShards), - "num_replicas" -> JsNumber(numReplicas), - "shards" -> JsObject(shardsAllocation) - )) - } - - private def unhealthyIndex(shardAllocation: Seq[(String, JsArray)]): Boolean = - shardAllocation.exists { - case ("unassigned", _) => true - case (_, JsArray(shards)) => shards.exists(!_.\("state").as[String].equals("STARTED")) - } - -} diff --git a/app/models/overview/Node.scala b/app/models/overview/Node.scala deleted file mode 100644 index 0086ee5..0000000 --- a/app/models/overview/Node.scala +++ /dev/null @@ -1,67 +0,0 @@ -package models.overview - -import models.commons.NodeRoles -import play.api.libs.json._ - -object Node { - - def apply(id: String, info: JsValue, stats: JsValue, masterNodeId: String) = { - val nodeRoles = NodeRoles(info) - - - // AWS nodes return no host/ip info - val host = (info \ "host").asOpt[JsString].getOrElse(JsNull) - val ip = (info \ "ip").asOpt[JsString].getOrElse(JsNull) - val jvmVersion = (info \ "jvm" \ "version").asOpt[JsString].getOrElse(JsNull) - - Json.obj( - "id" -> JsString(id), - "current_master" -> JsBoolean(id.equals(masterNodeId)), - "name" -> (info \ "name").as[JsString], - "host" -> host, - "ip" -> ip, - "es_version" -> (info \ "version").as[JsString], - "jvm_version" -> jvmVersion, - "load_average" -> loadAverage(stats), - "available_processors" -> (info \ "os" \ "available_processors").as[JsNumber], - "cpu_percent" -> cpuPercent(stats), - "master" -> JsBoolean(nodeRoles.master), - "data" -> JsBoolean(nodeRoles.data), - "coordinating" -> JsBoolean(nodeRoles.coordinating), - "ingest" -> JsBoolean(nodeRoles.ingest), - "heap" -> Json.obj( - "used" -> (stats \ "jvm" \ "mem" \ "heap_used_in_bytes").as[JsNumber], - "committed" -> (stats \ "jvm" \ "mem" \ "heap_committed_in_bytes").as[JsNumber], - "used_percent" -> (stats \ "jvm" \ "mem" \ "heap_used_percent").as[JsNumber], - "max" -> (stats \ "jvm" \ "mem" \ "heap_max_in_bytes").as[JsNumber] - ), - "disk" -> disk(stats) - ) - } - - def disk(stats: JsValue): JsObject = { - val totalInBytes = (stats \ "fs" \ "total" \ "total_in_bytes").asOpt[Long].getOrElse(0l) - val freeInBytes = (stats \ "fs" \ "total" \ "free_in_bytes").asOpt[Long].getOrElse(0l) - val usedPercent = 100 - (100 * (freeInBytes.toFloat / totalInBytes.toFloat)).toInt - Json.obj( - "total" -> JsNumber(totalInBytes), - "free" -> JsNumber(freeInBytes), - "used_percent" -> JsNumber(usedPercent) - ) - } - - def loadAverage(nodeStats: JsValue): JsNumber = { - val load = (nodeStats \ "os" \ "cpu" \ "load_average" \ "1m").asOpt[Float].getOrElse(// 5.X - (nodeStats \ "os" \ "load_average").asOpt[Float].getOrElse(0f) // FIXME: 2.X - ) - JsNumber(BigDecimal(load.toDouble)) - } - - def cpuPercent(nodeStats: JsValue): JsNumber = { - val cpu = (nodeStats \ "os" \ "cpu" \ "percent").asOpt[Int].getOrElse(// 5.X - (nodeStats \ "os" \ "cpu_percent").asOpt[Int].getOrElse(0) // FIXME 2.X - ) - JsNumber(BigDecimal(cpu)) - } - -} diff --git a/app/services/overview/ClusterOverview.scala b/app/services/overview/ClusterOverview.scala new file mode 100644 index 0000000..fc9a020 --- /dev/null +++ b/app/services/overview/ClusterOverview.scala @@ -0,0 +1,87 @@ +package services.overview + +import _root_.util.DataSize +import play.api.libs.json._ + +object ClusterOverview { + + def apply(clusterSettings: JsValue, health: JsValue, + nodes: JsValue, indices: JsValue, shards: JsValue, aliases: JsValue): JsValue = { + + val persistentAllocation = (clusterSettings \ "persistent" \ "cluster" \ "routing" \ "allocation" \ "enable").asOpt[String].getOrElse("all") + val transientAllocation = (clusterSettings \ "transient" \ "cluster" \ "routing" \ "allocation" \ "enable").asOpt[String] + val shardAllocation = transientAllocation.getOrElse(persistentAllocation).equals("all") + + def updateShardState(state: String) = __.json.update( + __.read[JsObject].map{ o => o ++ Json.obj( "state" -> JsString(state) ) } + ) + + val shardMap = (shards.as[JsArray].value.flatMap { + case shard => + val index = (shard \ "index").as[String] + val node = (shard \ "node").asOpt[String].getOrElse("unassigned") + // TODO: prune node/index from shard? + if ((shard \ "state").as[String].equals("RELOCATING")) { + val relocation = node.split(" ") + val origin = relocation.head + val target = relocation.last + val initializingShard = shard.transform(updateShardState("INITIALIZING")) match { + case JsSuccess(data, _) => data + case JsError(error) => throw new RuntimeException("boom") // TODO proper exception + } + Seq( + (index -> (origin -> shard)), + (index -> (target -> initializingShard)) + ) + } else { + Seq(index -> (node -> shard)) + } + }).groupBy(_._1).mapValues(v => v.map(_._2).groupBy(_._1).mapValues(v => JsArray(v.map(_._2)))) + + val aliasesMap = aliases.as[JsArray].value.map { + alias => (alias \ "index").as[String] -> (alias \ "alias").as[JsValue] + }.groupBy(_._1).mapValues(a => JsArray(a.map(_._2))) + + + + def addAliasAndShards(index: String) = __.json.update { + val aliases = JsObject(Map("aliases" -> aliasesMap.getOrElse(index, JsNull))) + val shards = JsObject(Map("shards" -> JsObject(shardMap.getOrElse(index, Map())))) + __.read[JsObject].map { o => o ++ aliases ++ shards } + } + + var sizeInBytes = 0l + var docsCount = 0 + var closedIndices = 0 + var specialIndices = 0 + + val withIndices = indices.as[JsArray].value.map { index => + val indexName = (index \ "index").as[String] + // TODO if ES returned a number + docsCount = docsCount + (index \ "docs.count").asOpt[String].map(_.toInt).getOrElse(0) + // TODO if ES returned in bytes... + sizeInBytes = sizeInBytes + (index \ "store.size").asOpt[String].map(DataSize.apply).getOrElse(0l) + if ((index \ "status").as[String].equals("close")) + closedIndices = closedIndices + 1 + if (indexName.startsWith(".")) + specialIndices = specialIndices + 1 + + index.transform(addAliasAndShards(indexName)) match { + case JsSuccess(data, _) => data + case JsError(error) => throw new RuntimeException("boom") // TODO proper exception + } + } + + JsObject(Seq( + "health" -> health, + "indices" -> JsArray(withIndices), + "nodes" -> nodes, + "shard_allocation" -> JsBoolean(shardAllocation), + "docs_count" -> JsNumber(BigDecimal(docsCount)), + "size_in_bytes" -> JsNumber(BigDecimal(sizeInBytes)), + "closed_indices" -> JsNumber(BigDecimal(closedIndices)), + "special_indices" -> JsNumber(BigDecimal(specialIndices)) + )) + } + +} diff --git a/app/services/overview/DataService.scala b/app/services/overview/DataService.scala new file mode 100644 index 0000000..f5cd58f --- /dev/null +++ b/app/services/overview/DataService.scala @@ -0,0 +1,62 @@ +package services.overview + +import com.google.inject.Inject +import elastic.{ElasticClient, Error} +import models.ElasticServer +import play.api.libs.json.JsValue +import services.exception.RequestFailedException + +import scala.concurrent.ExecutionContext.Implicits.global +import scala.concurrent.Future + +class DataService @Inject()(client: ElasticClient) { + + val NodesHeaders = Seq( + "id", + "ip", + "version", + "jdk", + "disk.total", + "disk.avail", + "disk.used_percent", + "heap.current", + "heap.percent", + "heap.max", + "cpu", + "load_1m", + "node.role", + "master", + "name" + ) + + val apis = Seq( + "_cluster/settings", + "_cluster/health", + s"_cat/nodes?h=${NodesHeaders.mkString(",")}&format=json", + "_cat/indices?format=json", + "_cat/shards?format=json&h=index,shard,prirep,node,state", + "_cat/aliases?h=alias,index&format=json" + ) + + def getOverviewData(target: ElasticServer): Future[JsValue] = { + Future.sequence( + apis.map(client.executeRequest("GET", _, None, target)) + ).map { responses => + responses.zipWithIndex.find(_._1.isInstanceOf[Error]) match { + case Some((response, idx)) => + throw RequestFailedException(apis(idx), response.status, response.body.toString) + + case None => + ClusterOverview( + responses(0).body, + responses(1).body, + responses(2).body, + responses(3).body, + responses(4).body, + responses(5).body + ) + } + } + } + +} diff --git a/app/services/overview/Index.scala b/app/services/overview/Index.scala new file mode 100644 index 0000000..1f43dba --- /dev/null +++ b/app/services/overview/Index.scala @@ -0,0 +1,44 @@ +package services.overview + +import play.api.libs.json._ + +object Index { + + def apply(data: JsValue, shardsData: Seq[JsValue], aliases: Seq[JsString]): JsValue = { + + val special = (data \ "index").as[String].startsWith(".") + + val shards = shardsData.map { + case shard => + val node = (shard \ "node").asOpt[String].getOrElse("unassigned") + node -> Json.obj( + "shard" -> (shard \ "shard").as[JsValue], + "primary" -> (shard \ "prirep").asOpt[String].contains("p"), + "index" -> (shard \ "index").as[JsValue], + "state" -> (shard \ "state").as[JsValue] + ) + }.groupBy(_._1).mapValues(v => JsArray(v.map(_._2))) + + JsObject(Seq( + "name" -> (data \ "index").as[JsValue], + "closed" -> JsBoolean((data \ "status").as[String].equals("close")), + "special" -> JsBoolean(special), + "unhealthy" -> JsBoolean(unhealthyIndex(shards)), + "doc_count" -> (data \ "docs.count").as[JsValue], + "deleted_docs" -> (data \ "docs.deleted").as[JsValue], + "size_in_bytes" -> (data \ "pri.store.size").as[JsValue], + "total_size_in_bytes" -> (data \ "store.size").as[JsValue], + "aliases" -> JsArray(aliases), + "num_shards" -> (data \ "pri").as[JsValue], + "num_replicas" -> (data \ "rep").as[JsValue], + "shards" -> JsObject(shards) + )) + } + + private def unhealthyIndex(shards: Map[String, JsArray]): Boolean = + shards.exists { + case ("unassigned", _) => true + case (_, JsArray(s)) => s.exists(!_.\("state").as[String].equals("STARTED")) + } + +} diff --git a/app/services/overview/Node.scala b/app/services/overview/Node.scala new file mode 100644 index 0000000..2835683 --- /dev/null +++ b/app/services/overview/Node.scala @@ -0,0 +1,47 @@ +package services.overview + +import models.commons.NodeRoles +import play.api.libs.json._ + +object Node { + + def apply(data: JsValue) = { + + val nodeRoles = NodeRoles((data \ "node.role").as[String]) + + // Not available on AWS + val ip = (data \ "ip").asOpt[JsValue].getOrElse(JsNull) + val jdk = (data \ "jdk").asOpt[JsValue].getOrElse(JsNull) + + // Not available on ES < 5.5 + val load = (data \ "load_1m").asOpt[JsValue].getOrElse(JsNull) + val diskTotal = (data \ "disk.total").asOpt[JsValue].getOrElse(JsNull) + val diskPercent = (data \ "disk.used_percent").asOpt[JsValue].getOrElse(JsNull) + + Json.obj( + "id" -> (data \ "id").as[JsValue], + "current_master" -> JsBoolean((data \ "master").as[String].equals("*")), + "name" -> (data \ "name").as[JsValue], + "ip" -> ip, + "jvm_version" -> jdk, + "es_version" -> (data\ "version").as[JsValue], + "load_1m" -> load, + "cpu" -> (data \ "cpu").as[JsValue], + "master" -> JsBoolean(nodeRoles.master), + "data" -> JsBoolean(nodeRoles.data), + "coordinating" -> JsBoolean(nodeRoles.coordinating), + "ingest" -> JsBoolean(nodeRoles.ingest), + "heap" -> Json.obj( + "current" -> (data \ "heap.current").as[JsValue], + "percent" -> (data \ "heap.percent").as[JsValue], + "max" -> (data \ "heap.max").as[JsValue] + ), + "disk" -> Json.obj( + "total" -> diskTotal, + "avail" -> (data \ "disk.avail").as[JsValue], + "percent" -> diskPercent + ) + ) + } + +} diff --git a/app/util/DataSize.scala b/app/util/DataSize.scala new file mode 100644 index 0000000..3110950 --- /dev/null +++ b/app/util/DataSize.scala @@ -0,0 +1,20 @@ +package util + +object DataSize { + + final val Units = Seq( + ("pb" -> 1024 * 1024 * 1024 * 1024 * 1024), + ("tb" -> 1024 * 1024 * 1024 * 1024), + ("gb" -> 1024 * 1024 * 1024), + ("mb" -> 1024 * 1024), + ("kb" -> 1024), + ("b" -> 1) + ) + + def apply(size: String): Long = { + Units.collectFirst { + case unit if (size.endsWith(unit._1)) => (size.substring(0, size.indexOf(unit._1)).toFloat * unit._2).toLong + }.getOrElse(0l) + } + +} diff --git a/public/js/app.js b/public/js/app.js index eb3c0a4..5c5d9f5 100644 --- a/public/js/app.js +++ b/public/js/app.js @@ -811,20 +811,18 @@ angular.module('cerebro').controller('OverviewController', ['$scope', '$http', function($scope, $http, $window, $location, OverviewDataService, AlertService, ModalService, RefreshService) { - $scope.data = undefined; - - $scope.indices = undefined; - $scope.nodes = undefined; - $scope.unassigned_shards = 0; - $scope.relocating_shards = 0; - $scope.initializing_shards = 0; - $scope.closed_indices = 0; - $scope.special_indices = 0; + $scope.data = undefined; // raw response + $scope.indices = undefined; // visible indices + $scope.nodes = undefined; // visible nodes $scope.shardAllocation = true; $scope.indices_filter = new IndexFilter('', false, false, true, true, 0); $scope.nodes_filter = new NodeFilter('', true, false, false, false, 0); + $scope.shardAsInt = function(shard) { // TODO if ES returned shard as Int... + return parseInt(shard.shard); + }; + $scope.getPageSize = function() { return Math.max(Math.round($window.innerWidth / 280), 1); }; @@ -833,8 +831,8 @@ angular.module('cerebro').controller('OverviewController', ['$scope', '$http', 1, $scope.getPageSize(), [], - $scope.indices_filter) - ; + $scope.indices_filter + ); $scope.page = $scope.paginator.getPage(); @@ -860,11 +858,6 @@ angular.module('cerebro').controller('OverviewController', ['$scope', '$http', $scope.data = data; $scope.setIndices(data.indices); $scope.setNodes(data.nodes); - $scope.unassigned_shards = data.unassigned_shards; - $scope.relocating_shards = data.relocating_shards; - $scope.initializing_shards = data.initializing_shards; - $scope.closed_indices = data.closed_indices; - $scope.special_indices = data.special_indices; $scope.shardAllocation = data.shard_allocation; }, function(error) { @@ -872,11 +865,6 @@ angular.module('cerebro').controller('OverviewController', ['$scope', '$http', $scope.data = undefined; $scope.indices = undefined; $scope.nodes = undefined; - $scope.unassigned_shards = 0; - $scope.relocating_shards = 0; - $scope.initializing_shards = 0; - $scope.closed_indices = 0; - $scope.special_indices = 0; $scope.shardAllocation = true; } ); @@ -1108,7 +1096,7 @@ angular.module('cerebro').controller('OverviewController', ['$scope', '$http', $scope.relocateShard = function(node) { var s = $scope.relocatingShard; - OverviewDataService.relocateShard(s.shard, s.index, s.node, node.id, + OverviewDataService.relocateShard(s.shard, s.index, s.node, node.name, function(response) { $scope.relocatingShard = undefined; RefreshService.refresh(); @@ -1129,8 +1117,8 @@ angular.module('cerebro').controller('OverviewController', ['$scope', '$http', $scope.canReceiveShard = function(index, node) { var shard = $scope.relocatingShard; if (shard && index) { // in case num indices < num columns - if (shard.node !== node.id && shard.index === index.name) { - var shards = index.shards[node.id]; + if (shard.node !== node.name && shard.index === index.index) { + var shards = index.shards[node.name]; if (shards) { var sameShard = function(s) { return s.shard === shard.shard; @@ -1969,9 +1957,9 @@ function IndexFilter(name, closed, special, healthy, asc, timestamp) { case 'name': return function(a, b) { if (asc) { - return a.name.localeCompare(b.name); + return a.index.localeCompare(b.index); } else { - return b.name.localeCompare(a.name); + return b.index.localeCompare(a.index); } }; default: @@ -2014,10 +2002,10 @@ function IndexFilter(name, closed, special, healthy, asc, timestamp) { this.matches = function(index) { var matches = true; - if (!this.special && index.special) { + if (!this.special && index.index.indexOf('.') === 0) { matches = false; } - if (!this.closed && index.closed) { + if (!this.closed && index.status === 'close') { matches = false; } // Hide healthy == show unhealthy only @@ -2027,7 +2015,7 @@ function IndexFilter(name, closed, special, healthy, asc, timestamp) { if (matches && this.name) { try { var regExp = new RegExp(this.name.trim(), 'i'); - matches = regExp.test(index.name); + matches = regExp.test(index.index); if (!matches && index.aliases) { for (var idx = 0; idx < index.aliases.length; idx++) { if ((matches = regExp.test(index.aliases[idx]))) { @@ -2037,7 +2025,7 @@ function IndexFilter(name, closed, special, healthy, asc, timestamp) { } } catch (err) { // if not valid regexp, still try normal matching - matches = index.name.indexOf(this.name.toLowerCase()) != -1; + matches = index.index.indexOf(this.name.toLowerCase()) != -1; if (!matches) { for (var _idx = 0; _idx < index.aliases.length; _idx++) { var alias = index.aliases[_idx].toLowerCase(); @@ -2099,10 +2087,10 @@ function NodeFilter(name, data, master, ingest, coordinating, timestamp) { this.matchesType = function(node) { return ( - node.data && this.data || - node.master && this.master || - node.ingest && this.ingest || - node.coordinating && this.coordinating + node['node.role'].indexOf('d') !== -1 && this.data || + node['node.role'].indexOf('m') !== -1 && this.master || + node['node.role'].indexOf('i') !== -1 && this.ingest || + node['node.role'].indexOf('-') !== -1 && this.coordinating ); }; diff --git a/public/overview.html b/public/overview.html index cf9fee1..f66f9e2 100644 --- a/public/overview.html +++ b/public/overview.html @@ -11,14 +11,14 @@
@@ -95,48 +95,48 @@ - +
- shards: {{index.num_shards}} * {{index.num_replicas + 1}}| - docs: {{index.doc_count | number}} | - size: {{index.size_in_bytes | bytes}} + shards: {{index.pri}} * {{index.rep}}| + docs: {{index['docs.count']}} | + size: {{index['store.size']}} index closed @@ -162,16 +162,16 @@ - + -
- {{unassigned_shards}} unassigned shards +
+ {{data.health.unassigned_shards}} unassigned shards
-
- {{relocating_shards}} relocating shards +
+ {{data.health.relocating_shards}} relocating shards
-
- {{initializing_shards}} initializing shards +
+ {{data.health.initializing_shards}} initializing shards
show only affected indices @@ -179,7 +179,7 @@
- + {{shard.shard}} @@ -191,81 +191,71 @@
-
- - +
+ +
-
+
-
+
- {{node.name}}
- {{node.host}} + {{node.ip}} +
+
+ + Disk avail.: {{node['disk.avail']}} + Load 1m: {{node.load_1m}} +
-
+
-
+
-
-
- -
-
-
- JVM: {{node.jvm_version}} - ES: {{node.es_version}} + JVM: {{node.jdk}} + ES: {{node.version}}
- + + id="{{shard.shard}}_{{node.name}}_{{index.index}}"> {{shard.shard}}