I am not sure whether the given sample config is valid or not - https://raw.githubusercontent.com/snowplow/snowplow-elasticsearch-loader/master/examples/config.hocon.sample
Elastic loader version: snowplow-elasticsearch-loader-http-0.10.1.jar
I faced configuration error related to kinesis & nsq. Solved that by adding kinesis & nsq section.
Following is the elastic loader configuration I am using -
########################################################################
source = kinesis
sink {
good = "elasticsearch"
bad = "stderr"
}
enabled = "good"
aws {
accessKey = XXXXXXXXXXXXXXXXXXX
secretKey = "XXXXXXXXXXXXXXXXXXX"
}
kinesis {
initialPosition = "LATEST"
initialTimestamp = "{{initialTimestamp}}
maxRecords = 10000**
region = "us-east-1"
appName = "ELK-Loader"
}
nsq {
channelName = "dummy"
nsqdHost = "dummy"
nsqdPort = 0
nsqlookupdHost = "dummy"
nsqlookupdPort = 0
}
queue {
enabled="kinesis"
initialPosition = "{{initialPosition}}"
initialTimestamp = "{{initialTimestamp}}"
maxRecords = 10000
region = "us-east-1"
appName = "ELK-Loader"
channelName = "dummy"
nsqdHost = "dummy"
nsqdPort = 0
nsqlookupdHost = "dummy"
nsqlookupdPort = 0
}
streams {
inStreamName = "rr-dev-snowplow-enriched-good"
outStreamName = "rr-dev-snowplow-enriched-bad"
buffer {
byteLimit = 4500000 # Not supported by NSQ, will be ignored
recordLimit = 500
timeLimit = 60000 # Not supported by NSQ, will be ignored
}
}
elasticsearch {
client {
endpoint = "https://search-rr-snowplow-events-3ks5yicuiihikty5mhvl3tdwuq.us-east-1.es.amazonaws.com"
port = "9200"
username = "XXXXXXXXXX"
password = "XXXXXXXXXX"
shardDateFormat = "{{elasticsearchShardDateFormat}}"
shardDateField = "{{elasticsearchShardDateField}}"
maxTimeout = "6000"
maxRetries = 5
ssl = false
}
aws {
signing = false
region = "us-east-1"
}
cluster {
name = "135947549772:rr-snowplow-events"
index = "snowplow"
clusterType = "enriched"
}
}
####################################################################
Also facing the following error on running -
$ java -jar snowplow-elasticsearch-loader-http-0.10.2.jar --config rr_snowplow_elk_loader.conf
Exception in thread "main" java.lang.RuntimeException: Invalid hosts/ports https://search-rr-snowplow-events-3ks5yicuiihikty5mhvl3tdwuq.us-east-1.es.amazonaws.com:9200
at scala.sys.package$.error(package.scala:27)
at com.sksamuel.elastic4s.ElasticsearchClientUri$$anonfun$3.apply(ElasticsearchClientUri.scala:23)
at com.sksamuel.elastic4s.ElasticsearchClientUri$$anonfun$3.apply(ElasticsearchClientUri.scala:21)
at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
at scala.collection.IndexedSeqOptimized$class.foreach(IndexedSeqOptimized.scala:33)
at scala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:186)
at scala.collection.TraversableLike$class.map(TraversableLike.scala:234)
at scala.collection.mutable.ArrayOps$ofRef.map(ArrayOps.scala:186)
at com.sksamuel.elastic4s.ElasticsearchClientUri$.apply(ElasticsearchClientUri.scala:21)
at com.snowplowanalytics.elasticsearch.loader.clients.ElasticsearchSenderHTTP.<init>(ElasticsearchSenderHTTP.scala:55)
at com.snowplowanalytics.elasticsearch.loader.ElasticsearchHTTPSinkApp$.elasticsearchSender$lzycompute(ElasticsearchHTTPSinkApp.scala:39)
at com.snowplowanalytics.elasticsearch.loader.ElasticsearchHTTPSinkApp$.elasticsearchSender(ElasticsearchHTTPSinkApp.scala:38)
at com.snowplowanalytics.elasticsearch.loader.ElasticsearchSinkApp$class.run(ElasticsearchSinkApp.scala:168)
at com.snowplowanalytics.elasticsearch.loader.ElasticsearchHTTPSinkApp$.run(ElasticsearchHTTPSinkApp.scala:25)
at com.snowplowanalytics.elasticsearch.loader.ElasticsearchHTTPSinkApp$.delayedEndpoint$com$snowplowanalytics$elasticsearch$loader$ElasticsearchHTTPSinkApp$1(ElasticsearchHTTPSinkApp.scala:41)
at com.snowplowanalytics.elasticsearch.loader.ElasticsearchHTTPSinkApp$delayedInit$body.apply(ElasticsearchHTTPSinkApp.scala:25)
at scala.Function0$class.apply$mcV$sp(Function0.scala:34)
at scala.runtime.AbstractFunction0.apply$mcV$sp(AbstractFunction0.scala:12)
at scala.App$$anonfun$main$1.apply(App.scala:76)
at scala.App$$anonfun$main$1.apply(App.scala:76)
at scala.collection.immutable.List.foreach(List.scala:392)
at scala.collection.generic.TraversableForwarder$class.foreach(TraversableForwarder.scala:35)
at scala.App$class.main(App.scala:76)
at com.snowplowanalytics.elasticsearch.loader.ElasticsearchHTTPSinkApp$.main(ElasticsearchHTTPSinkApp.scala:25)
at com.snowplowanalytics.elasticsearch.loader.ElasticsearchHTTPSinkApp.main(ElasticsearchHTTPSinkApp.scala)