Skip to content
Snippets Groups Projects
Commit 6a414ee0 authored by Leonardo Menezes's avatar Leonardo Menezes
Browse files

fixes missing initializing shard when relocating shard exists 

closes #250
parent 5973a17f
No related branches found
No related tags found
No related merge requests found
Show whitespace changes
Inline Side-by-side
app/models/overview/Index.scala
+ 76
− 27
View file @ 6a414ee0
...@@ -4,39 +4,88 @@ import play.api.libs.json._ ...@@ -4,39 +4,88 @@ import play.api.libs.json._
object Index { object Index {
def apply(name: String, stats: JsValue, routingTable: JsValue, aliases: JsObject): JsValue = { def apply(name: String, stats: JsValue, routingTable: JsValue, aliases: JsValue): JsValue = {
var unhealthy = false val shardMap = createShardMap(routingTable)
var numReplicas = 0
var numShards = 0
val shardMap = (routingTable \ "shards").as[JsObject].value.toSeq.flatMap { case (num, shards) =>
numShards = Math.max(numShards, num.toInt + 1) // shard num is 0 based
val shardInstances = shards.as[JsArray].value
numReplicas = shardInstances.length - 1
shardInstances.map { shard =>
unhealthy = unhealthy || !(shard \ "state").as[String].equals("STARTED")
(shard \ "node").asOpt[String].getOrElse("unassigned") -> shard
}
}.groupBy(_._1).mapValues(v => JsArray(v.map(_._2)))
val special = name.startsWith(".")
JsObject(Seq( JsObject(Seq(
"name" -> JsString(name), "name" -> JsString(name),
"closed" -> JsBoolean(false), "closed" -> JsBoolean(false),
"special" -> JsBoolean(special), "special" -> JsBoolean(name.startsWith(".")),
"unhealthy" -> JsBoolean(unhealthy), "unhealthy" -> JsBoolean(isIndexUnhealthy(shardMap)),
"doc_count" -> (stats \ "primaries" \ "docs" \ "count").asOpt[JsNumber].getOrElse(JsNumber(0)), "doc_count" -> (stats \ "primaries" \ "docs" \ "count").asOpt[JsNumber].getOrElse(JsNumber(0)),
"deleted_docs" -> (stats \ "primaries" \ "docs" \ "deleted").asOpt[JsNumber].getOrElse(JsNumber(0)), "deleted_docs" -> (stats \ "primaries" \ "docs" \ "deleted").asOpt[JsNumber].getOrElse(JsNumber(0)),
"size_in_bytes" -> (stats \ "primaries" \ "store" \ "size_in_bytes").asOpt[JsNumber].getOrElse(JsNumber(0)), "size_in_bytes" -> (stats \ "primaries" \ "store" \ "size_in_bytes").asOpt[JsNumber].getOrElse(JsNumber(0)),
"total_size_in_bytes" -> (stats \ "total" \ "store" \ "size_in_bytes").asOpt[JsNumber].getOrElse(JsNumber(0)), "total_size_in_bytes" -> (stats \ "total" \ "store" \ "size_in_bytes").asOpt[JsNumber].getOrElse(JsNumber(0)),
"aliases" -> JsArray(aliases.keys.map(JsString(_)).toSeq), // 1.4 < does not return aliases obj "aliases" -> JsArray(aliases.as[JsObject].keys.map(JsString(_)).toSeq), // 1.4 < does not return aliases obj
"num_shards" -> JsNumber(numShards), "num_shards" -> JsNumber((routingTable \ "shards").as[JsObject].keys.map(_.toInt).max + 1),
"num_replicas" -> JsNumber(numReplicas), "num_replicas" -> JsNumber((routingTable \ "shards" \ "0").as[JsArray].value.size - 1),
"shards" -> JsObject(shardMap) "shards" -> JsObject(shardMap)
)) ))
} }
/**
* Parses shard information as a pair where first element is the allocating node and value
* is the shard representation itself.
*
* For relocating shards, creates an extra shard with INITIALIZING state allocated to the target node
*
* @param shard
* @return
*/
def parseShard(shard: JsValue): Seq[(String, JsValue)] = {
Seq((shard \ "node").asOpt[String].getOrElse("unassigned") -> shard) ++
(shard \ "relocating_node").asOpt[String].map(createInitializingShard(_, shard)).toSeq
}
/**
* Transforms the routing table in a map where the keys are node ids and the values are
* the list of shards allocated to the given node. For unassigned shards, unassigned is used
* as the key.
*
* @param routingTable
* @return
*/
def createShardMap(routingTable: JsValue): Map[String, JsArray] = {
(routingTable \ "shards").as[JsObject].value.toSeq.flatMap { case (_, shards) =>
shards.as[JsArray].value.flatMap(parseShard(_))
}.groupBy(_._1).mapValues(v => JsArray(v.map(_._2)))
}
/**
* Returns true if at least one of the index shards is considered unhealthy
*
* @param shardMap
* @return
*/
private def isIndexUnhealthy(shardMap: Map[String, JsArray]): Boolean =
shardMap.values.exists { shards =>
shards.value.exists { shard => isShardUnhealthy(shard) }
}
/**
* Returns true if the shard state is other than STARTED
*
* @param shard
* @return
*/
private def isShardUnhealthy(shard: JsValue): Boolean =
!(shard \ "state").as[String].equals("STARTED")
/**
* Creates an artificial shard with initializing state allocated to the target node
*
* @param targetNode
* @param shard
* @return
*/
private def createInitializingShard(targetNode: String, shard: JsValue): (String, JsValue) =
(targetNode -> Json.obj(
"node" -> JsString(targetNode),
"index" -> (shard \ "index").as[JsValue],
"state" -> JsString("INITIALIZING"),
"shard" -> (shard \ "shard").as[JsValue],
"primary" -> JsBoolean(false)
))
} }
test/models/overview/IndexAliases.scala 0 → 100644
+ 15
− 0
View file @ 6a414ee0
package models.overview
import play.api.libs.json.Json
object IndexAliases {
val aliases = Json.parse(
"""
|{
| "fancyAlias": {}
|}
""".stripMargin
)
}
test/models/overview/IndexRoutingTable.scala 0 → 100644
+ 237
− 0
View file @ 6a414ee0
package models.overview
import play.api.libs.json.Json
object IndexRoutingTable {
val healthyShards = Json.parse(
"""
|{
| "shards": {
| "0": [
| {
| "state": "STARTED",
| "primary": true,
| "node": "ZqGi3UPESiSa0Z4Sf4NlPg",
| "relocating_node": null,
| "shard": 0,
| "index": "some",
| "allocation_id": {
| "id": "LEm_TRI3TFuH3icSnvkvQg"
| }
| }
| ],
| "1": [
| {
| "state": "STARTED",
| "primary": true,
| "node": "ZqGi3UPESiSa0Z4Sf4NlPg",
| "relocating_node": null,
| "shard": 1,
| "index": "some",
| "allocation_id": {
| "id": "YUY5QiPmQJulsereqC1VBQ"
| }
| }
| ],
| "2": [
| {
| "state": "STARTED",
| "primary": true,
| "node": "H-4gqX87SYqmQKtsatg92w",
| "relocating_node": null,
| "shard": 2,
| "index": "some",
| "allocation_id": {
| "id": "LXEh1othSz6IE5ueTITF-Q"
| }
| }
| ],
| "3": [
| {
| "state": "STARTED",
| "primary": true,
| "node": "H-4gqX87SYqmQKtsatg92w",
| "relocating_node": null,
| "shard": 3,
| "index": "some",
| "allocation_id": {
| "id": "6X6SMPvvQbOdUct5k3bo6w"
| }
| }
| ],
| "4": [
| {
| "state": "STARTED",
| "primary": true,
| "node": "ZqGi3UPESiSa0Z4Sf4NlPg",
| "relocating_node": null,
| "shard": 4,
| "index": "some",
| "allocation_id": {
| "id": "oWmBTuCFSuGA4krn5diK3w"
| }
| }
| ]
| }
|}
""".stripMargin
)
val relocatingShard = Json.parse(
"""
|{
| "shards": {
| "0": [
| {
| "state": "STARTED",
| "primary": true,
| "node": "ZqGi3UPESiSa0Z4Sf4NlPg",
| "relocating_node": null,
| "shard": 0,
| "index": "some",
| "allocation_id": {
| "id": "LEm_TRI3TFuH3icSnvkvQg"
| }
| }
| ],
| "1": [
| {
| "state": "STARTED",
| "primary": true,
| "node": "ZqGi3UPESiSa0Z4Sf4NlPg",
| "relocating_node": null,
| "shard": 1,
| "index": "some",
| "allocation_id": {
| "id": "YUY5QiPmQJulsereqC1VBQ"
| }
| }
| ],
| "2": [
| {
| "state": "STARTED",
| "primary": true,
| "node": "H-4gqX87SYqmQKtsatg92w",
| "relocating_node": null,
| "shard": 2,
| "index": "some",
| "allocation_id": {
| "id": "LXEh1othSz6IE5ueTITF-Q"
| }
| }
| ],
| "3": [
| {
| "state": "STARTED",
| "primary": true,
| "node": "H-4gqX87SYqmQKtsatg92w",
| "relocating_node": null,
| "shard": 3,
| "index": "some",
| "allocation_id": {
| "id": "6X6SMPvvQbOdUct5k3bo6w"
| }
| }
| ],
| "4": [
| {
| "state": "RELOCATING",
| "primary": true,
| "node": "ZqGi3UPESiSa0Z4Sf4NlPg",
| "relocating_node": "H-4gqX87SYqmQKtsatg92w",
| "shard": 4,
| "index": "some",
| "expected_shard_size_in_bytes": 32860995,
| "allocation_id": {
| "id": "oWmBTuCFSuGA4krn5diK3w",
| "relocation_id": "fSkjawIwQ7e2LuVGE9X1MQ"
| }
| }
| ]
| }
|}
""".stripMargin
)
val unassignedShard = Json.parse(
"""
|{
| "shards": {
| "0": [
| {
| "state": "STARTED",
| "primary": true,
| "node": "ZqGi3UPESiSa0Z4Sf4NlPg",
| "relocating_node": null,
| "shard": 0,
| "index": "some",
| "allocation_id": {
| "id": "LEm_TRI3TFuH3icSnvkvQg"
| }
| }
| ],
| "1": [
| {
| "state": "STARTED",
| "primary": true,
| "node": "ZqGi3UPESiSa0Z4Sf4NlPg",
| "relocating_node": null,
| "shard": 1,
| "index": "some",
| "allocation_id": {
| "id": "YUY5QiPmQJulsereqC1VBQ"
| }
| }
| ],
| "2": [
| {
| "state": "STARTED",
| "primary": true,
| "node": "H-4gqX87SYqmQKtsatg92w",
| "relocating_node": null,
| "shard": 2,
| "index": "some",
| "allocation_id": {
| "id": "LXEh1othSz6IE5ueTITF-Q"
| }
| }
| ],
| "3": [
| {
| "state": "STARTED",
| "primary": true,
| "node": "H-4gqX87SYqmQKtsatg92w",
| "relocating_node": null,
| "shard": 3,
| "index": "some",
| "allocation_id": {
| "id": "6X6SMPvvQbOdUct5k3bo6w"
| }
| }
| ],
| "4": [
| {
| "state": "UNASSIGNED",
| "primary": false,
| "node": null,
| "relocating_node": null,
| "shard": 4,
| "index": "some",
| "recovery_source": {
| "type": "PEER"
| },
| "unassigned_info": {
| "reason": "REPLICA_ADDED",
| "at": "2018-01-04T10:10:14.154Z",
| "delayed": false,
| "allocation_status": "no_attempt"
| }
| }
| ]
| }
|}
""".stripMargin
)
}
test/models/overview/IndexSpec.scala 0 → 100644
+ 376
− 0
View file @ 6a414ee0
package models.overview
import org.specs2.Specification
import play.api.libs.json.Json
object IndexSpec extends Specification {
def is =
s2"""
Index should
build a healthy index $healthy
build an index with relocating shards $relocating
build an index with unassigned shards $unassigned
build a special index $special
"""
def healthy = {
val expected = Json.parse(
"""
|{
| "name": "ipsum",
| "closed": false,
| "special": false,
| "unhealthy": false,
| "doc_count": 62064,
| "deleted_docs": 0,
| "size_in_bytes": 163291998,
| "total_size_in_bytes": 326583996,
| "aliases": [
| "fancyAlias"
| ],
| "num_shards": 5,
| "num_replicas": 0,
| "shards": {
| "ZqGi3UPESiSa0Z4Sf4NlPg": [
| {
| "state": "STARTED",
| "primary": true,
| "node": "ZqGi3UPESiSa0Z4Sf4NlPg",
| "relocating_node": null,
| "shard": 4,
| "index": "some",
| "allocation_id": {
| "id": "oWmBTuCFSuGA4krn5diK3w"
| }
| },
| {
| "state": "STARTED",
| "primary": true,
| "node": "ZqGi3UPESiSa0Z4Sf4NlPg",
| "relocating_node": null,
| "shard": 1,
| "index": "some",
| "allocation_id": {
| "id": "YUY5QiPmQJulsereqC1VBQ"
| }
| },
| {
| "state": "STARTED",
| "primary": true,
| "node": "ZqGi3UPESiSa0Z4Sf4NlPg",
| "relocating_node": null,
| "shard": 0,
| "index": "some",
| "allocation_id": {
| "id": "LEm_TRI3TFuH3icSnvkvQg"
| }
| }
| ],
| "H-4gqX87SYqmQKtsatg92w": [
| {
| "state": "STARTED",
| "primary": true,
| "node": "H-4gqX87SYqmQKtsatg92w",
| "relocating_node": null,
| "shard": 2,
| "index": "some",
| "allocation_id": {
| "id": "LXEh1othSz6IE5ueTITF-Q"
| }
| },
| {
| "state": "STARTED",
| "primary": true,
| "node": "H-4gqX87SYqmQKtsatg92w",
| "relocating_node": null,
| "shard": 3,
| "index": "some",
| "allocation_id": {
| "id": "6X6SMPvvQbOdUct5k3bo6w"
| }
| }
| ]
| }
|}
""".stripMargin
)
val index = Index("ipsum", IndexStats.stats, IndexRoutingTable.healthyShards, IndexAliases.aliases)
index mustEqual expected
}
def relocating = {
val expected = Json.parse(
"""
|{
| "name": "ipsum",
| "closed": false,
| "special": false,
| "unhealthy": true,
| "doc_count": 62064,
| "deleted_docs": 0,
| "size_in_bytes": 163291998,
| "total_size_in_bytes": 326583996,
| "aliases": [
| "fancyAlias"
| ],
| "num_shards": 5,
| "num_replicas": 0,
| "shards": {
| "ZqGi3UPESiSa0Z4Sf4NlPg": [
| {
| "state": "RELOCATING",
| "primary": true,
| "node": "ZqGi3UPESiSa0Z4Sf4NlPg",
| "relocating_node": "H-4gqX87SYqmQKtsatg92w",
| "shard": 4,
| "index": "some",
| "expected_shard_size_in_bytes": 32860995,
| "allocation_id": {
| "id": "oWmBTuCFSuGA4krn5diK3w",
| "relocation_id": "fSkjawIwQ7e2LuVGE9X1MQ"
| }
| },
| {
| "state": "STARTED",
| "primary": true,
| "node": "ZqGi3UPESiSa0Z4Sf4NlPg",
| "relocating_node": null,
| "shard": 1,
| "index": "some",
| "allocation_id": {
| "id": "YUY5QiPmQJulsereqC1VBQ"
| }
| },
| {
| "state": "STARTED",
| "primary": true,
| "node": "ZqGi3UPESiSa0Z4Sf4NlPg",
| "relocating_node": null,
| "shard": 0,
| "index": "some",
| "allocation_id": {
| "id": "LEm_TRI3TFuH3icSnvkvQg"
| }
| }
| ],
| "H-4gqX87SYqmQKtsatg92w": [
| {
| "node": "H-4gqX87SYqmQKtsatg92w",
| "index": "some",
| "state": "INITIALIZING",
| "shard": 4,
| "primary": false
| },
| {
| "state": "STARTED",
| "primary": true,
| "node": "H-4gqX87SYqmQKtsatg92w",
| "relocating_node": null,
| "shard": 2,
| "index": "some",
| "allocation_id": {
| "id": "LXEh1othSz6IE5ueTITF-Q"
| }
| },
| {
| "state": "STARTED",
| "primary": true,
| "node": "H-4gqX87SYqmQKtsatg92w",
| "relocating_node": null,
| "shard": 3,
| "index": "some",
| "allocation_id": {
| "id": "6X6SMPvvQbOdUct5k3bo6w"
| }
| }
| ]
| }
|}
""".stripMargin
)
val index = Index("ipsum", IndexStats.stats, IndexRoutingTable.relocatingShard, IndexAliases.aliases)
index mustEqual expected
}
def unassigned = {
val expected = Json.parse(
"""
|{
| "name": "ipsum",
| "closed": false,
| "special": false,
| "unhealthy": true,
| "doc_count": 62064,
| "deleted_docs": 0,
| "size_in_bytes": 163291998,
| "total_size_in_bytes": 326583996,
| "aliases": [
| "fancyAlias"
| ],
| "num_shards": 5,
| "num_replicas": 0,
| "shards": {
| "ZqGi3UPESiSa0Z4Sf4NlPg": [
| {
| "state": "STARTED",
| "primary": true,
| "node": "ZqGi3UPESiSa0Z4Sf4NlPg",
| "relocating_node": null,
| "shard": 1,
| "index": "some",
| "allocation_id": {
| "id": "YUY5QiPmQJulsereqC1VBQ"
| }
| },
| {
| "state": "STARTED",
| "primary": true,
| "node": "ZqGi3UPESiSa0Z4Sf4NlPg",
| "relocating_node": null,
| "shard": 0,
| "index": "some",
| "allocation_id": {
| "id": "LEm_TRI3TFuH3icSnvkvQg"
| }
| }
| ],
| "unassigned": [
| {
| "state": "UNASSIGNED",
| "primary": false,
| "node": null,
| "relocating_node": null,
| "shard": 4,
| "index": "some",
| "recovery_source": {
| "type": "PEER"
| },
| "unassigned_info": {
| "reason": "REPLICA_ADDED",
| "at": "2018-01-04T10:10:14.154Z",
| "delayed": false,
| "allocation_status": "no_attempt"
| }
| }
| ],
| "H-4gqX87SYqmQKtsatg92w": [
| {
| "state": "STARTED",
| "primary": true,
| "node": "H-4gqX87SYqmQKtsatg92w",
| "relocating_node": null,
| "shard": 2,
| "index": "some",
| "allocation_id": {
| "id": "LXEh1othSz6IE5ueTITF-Q"
| }
| },
| {
| "state": "STARTED",
| "primary": true,
| "node": "H-4gqX87SYqmQKtsatg92w",
| "relocating_node": null,
| "shard": 3,
| "index": "some",
| "allocation_id": {
| "id": "6X6SMPvvQbOdUct5k3bo6w"
| }
| }
| ]
| }
|}
""".stripMargin
)
val index = Index("ipsum", IndexStats.stats, IndexRoutingTable.unassignedShard, IndexAliases.aliases)
index mustEqual expected
}
def special = {
val expected = Json.parse(
"""
|{
| "name": ".ipsum",
| "closed": false,
| "special": true,
| "unhealthy": false,
| "doc_count": 62064,
| "deleted_docs": 0,
| "size_in_bytes": 163291998,
| "total_size_in_bytes": 326583996,
| "aliases": [
| "fancyAlias"
| ],
| "num_shards": 5,
| "num_replicas": 0,
| "shards": {
| "ZqGi3UPESiSa0Z4Sf4NlPg": [
| {
| "state": "STARTED",
| "primary": true,
| "node": "ZqGi3UPESiSa0Z4Sf4NlPg",
| "relocating_node": null,
| "shard": 4,
| "index": "some",
| "allocation_id": {
| "id": "oWmBTuCFSuGA4krn5diK3w"
| }
| },
| {
| "state": "STARTED",
| "primary": true,
| "node": "ZqGi3UPESiSa0Z4Sf4NlPg",
| "relocating_node": null,
| "shard": 1,
| "index": "some",
| "allocation_id": {
| "id": "YUY5QiPmQJulsereqC1VBQ"
| }
| },
| {
| "state": "STARTED",
| "primary": true,
| "node": "ZqGi3UPESiSa0Z4Sf4NlPg",
| "relocating_node": null,
| "shard": 0,
| "index": "some",
| "allocation_id": {
| "id": "LEm_TRI3TFuH3icSnvkvQg"
| }
| }
| ],
| "H-4gqX87SYqmQKtsatg92w": [
| {
| "state": "STARTED",
| "primary": true,
| "node": "H-4gqX87SYqmQKtsatg92w",
| "relocating_node": null,
| "shard": 2,
| "index": "some",
| "allocation_id": {
| "id": "LXEh1othSz6IE5ueTITF-Q"
| }
| },
| {
| "state": "STARTED",
| "primary": true,
| "node": "H-4gqX87SYqmQKtsatg92w",
| "relocating_node": null,
| "shard": 3,
| "index": "some",
| "allocation_id": {
| "id": "6X6SMPvvQbOdUct5k3bo6w"
| }
| }
| ]
| }
|}
""".stripMargin
)
val index = Index(".ipsum", IndexStats.stats, IndexRoutingTable.healthyShards, IndexAliases.aliases)
index mustEqual expected
}
}
test/models/overview/IndexStats.scala 0 → 100644
+ 33
− 0
View file @ 6a414ee0
package models.overview
import play.api.libs.json.Json
object IndexStats {
val stats = Json.parse(
"""
|{
| "primaries": {
| "docs": {
| "count": 62064,
| "deleted": 0
| },
| "store": {
| "size_in_bytes": 163291998
| }
| },
| "total": {
| "docs": {
| "count": 124128,
| "deleted": 0
| },
| "store": {
| "size_in_bytes": 326583996
| }
| }
|}
""".stripMargin
)
}
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment