Hi all,
It would be very nice if someone could help me out!
I have read many of the posts of this topic but I cannot get my em-etl-runner file to work.
I am using snowplow_emr_r97_knossos
This is the error:
ReturnContractError (Contract violation for return value:
Expected: #<Contracts::Maybe:0x294ec2c1 @vals=[{:aws=>{:access_key_id=>String, :secret_access_key=>String, :s3=>{:region=>String, :buckets=>{:assets=>String, :jsonpath_assets=>#<Contracts::Maybe:0xc7a1192 @vals=[String, nil]>, :log=>String, :raw=>{:in=>#<Contracts::CollectionOf:0x4423fc17 @collection_class=Array, @contract=String>, :processing=>String, :archive=>String}, :enriched=>{:good=>String, :bad=>String, :errors=>#<Contracts::Maybe:0x486608e4 @vals=[String, nil]>, :archive=>#<Contracts::Maybe:0x22d1d5de @vals=[String, nil]>}, :shredded=>{:good=>String, :bad=>String, :errors=>#<Contracts::Maybe:0x3b90c899 @vals=[String, nil]>, :archive=>#<Contracts::Maybe:0x1e61d35c @vals=[String, nil]>}}}, :emr=>{:ami_version=>String, :region=>String, :jobflow_role=>String, :service_role=>String, :placement=>#<Contracts::Maybe:0x30014542 @vals=[String, nil]>, :ec2_subnet_id=>#<Contracts::Maybe:0x65fa805 @vals=[String, nil]>, :ec2_key_name=>String, :bootstrap=>#<Contracts::Maybe:0x2c5cab9e @vals=[#<Contracts::CollectionOf:0x22058915 @collection_class=Array, @contract=String>, nil]>, :software=>{:hbase=>#<Contracts::Maybe:0x564e4cf1 @vals=[String, nil]>, :lingual=>#<Contracts::Maybe:0x69f54525 @vals=[String, nil]>}, :jobflow=>{:job_name=>String, :master_instance_type=>String, :core_instance_count=>Contracts::Num, :core_instance_type=>String, :core_instance_ebs=>#<Contracts::Maybe:0x1afbfce5 @vals=[{:volume_size=>#<Proc:0x1ef2c762@uri:classloader:/emr-etl-runner/lib/snowplow-emr-etl-runner/contracts.rb:28 (lambda)>, :volume_type=>#<Proc:0x550002cd@uri:classloader:/emr-etl-runner/lib/snowplow-emr-etl-runner/contracts.rb:27 (lambda)>, :volume_iops=>#<Contracts::Maybe:0x73b80c53 @vals=[#<Proc:0x1ef2c762@uri:classloader:/emr-etl-runner/lib/snowplow-emr-etl-runner/contracts.rb:28 (lambda)>, nil]>, :ebs_optimized=>#<Contracts::Maybe:0x2fa916ab @vals=[Contracts::Bool, nil]>}, nil]>, :task_instance_count=>Contracts::Num, :task_instance_type=>String, :task_instance_bid=>#<Contracts::Maybe:0x797b930d @vals=[Contracts::Num, nil]>}, :additional_info=>#<Contracts::Maybe:0x297f2629 @vals=[String, nil]>, :bootstrap_failure_tries=>Contracts::Num, :configuration=>#<Contracts::Maybe:0x6114cd31 @vals=[#<Contracts::HashOf:0x6f3116ef @value=#<Contracts::HashOf:0x7fd8ddf0 @value=String, @key=Symbol>, @key=Symbol>, nil]>}}, :collectors=>{:format=>String}, :enrich=>{:versions=>{:spark_enrich=>String}, :continue_on_unexpected_error=>Contracts::Bool, :output_compression=>#<Proc:0x76d2f922@uri:classloader:/emr-etl-runner/lib/snowplow-emr-etl-runner/contracts.rb:26 (lambda)>}, :storage=>{:versions=>{:rdb_shredder=>String, :hadoop_elasticsearch=>String, :rdb_loader=>String}}, :monitoring=>{:tags=>#<Contracts::HashOf:0x4eed1516 @value=String, @key=Symbol>, :logging=>{:level=>String}, :snowplow=>#<Contracts::Maybe:0x19ba0a49 @vals=[{:method=>String, :collector=>String, :app_id=>String}, nil]>}}, nil]>,
Actual: {:aws=>{:access_key_id=>“####”, :secret_access_key=>“####”, :s3=>{:region=>“eu-central-1”, :buckets=>{:assets=>“s3://snowplow-hosted-assets”, :jsonpath_assets=>nil, :log=>“s3://snowplow-etl-emr/logs”, :raw=>{:in=>[“s3://snowplow-logs-gtm”], :processing=>“s3://swapfiets-snowplow-data/processing”, :archive=>“s3://swapfiets-snowplow-data/archive/raw”}, :enriched=>{:good=>“s3://swapfiets-snowplow-data/enriched/good”, :bad=>“s3://swapfiets-snowplow-data/enriched/bad”, :errors=>nil, :archive=>“s3://swapfiets-snowplow-data/archive/enriched”}, :shredded=>{:good=>“s3://swapfiets-snowplow-data/shredded/good”, :bad=>“s3://swapfiets-snowplow-data/shredded/bad”, :errors=>nil, :archive=>“s3://swapfiets-snowplow-data/archive/shredded”}}}, :emr=>{:ami_version=>“5.9.0”, :region=>“eu-central-1”, :jobflow_role=>“EMR_EC2_DefaultRole”, :service_role=>“EMR_DefaultRole”, :placement=>nil, :ec2_subnet_id=>“subnet-####”, :ec2_key_name=>“####”, :bootstrap=>, :software=>{:hbase=>nil, :lingual=>nil}, :jobflow=>{:job_name=>“Snowplow ETL”, :master_instance_type=>“m1.medium”, :core_instance_count=>2, :core_instance_type=>“m1.medium”, :core_instance_ebs=>{:volume_size=>100, :volume_type=>“gp2”, :volume_iops=>400, :ebs_optimized=>false}, :task_instance_count=>0, :task_instance_type=>“m1.medium”, :task_instance_bid=>0.015}, :additional_info=>nil, :bootstrap_failure_tries=>3, :configuration=>nil}}, :collectors=>{:format=>“cloudfront”}, :enrich=>{:versions=>{:spark_enrich=>“1.10.0”}, :continue_on_unexpected_error=>false, :output_compression=>“NONE”}, :storage=>{:versions=>{:rdb_shredder=>“0.13.0”, :hadoop_elasticsearch=>“0.1.0”, :rdb_loader=>“0.14.0”}}, :monitoring=>{:tags=>{}, :logging=>{:level=>“DEBUG”}, :snowplow=>{:method=>“get”, :collector=>“[####.cloudfront.net], :app_id=>nil}}}
Value guarded in: Snowplow::EmrEtlRunner::Cli::load_config
With Contract: Maybe, String, Bool => Maybe
At: uri:classloader:/emr-etl-runner/lib/snowplow-emr-etl-runner/cli.rb:202 ):
uri:classloader:/gems/contracts-0.11.0/lib/contracts.rb:45:in `block in Contract' uri:classloader:/gems/contracts-0.11.0/lib/contracts.rb:154:in` failure_callback’
uri:classloader:/gems/contracts-0.11.0/lib/contracts/call_with.rb:80:in `call_with' uri:classloader:/gems/contracts-0.11.0/lib/contracts/method_handler.rb:138:in` block in redefine_method’
uri:classloader:/emr-etl-runner/lib/snowplow-emr-etl-runner/cli.rb:191:in `process_options' uri:classloader:/emr-etl-runner/lib/snowplow-emr-etl-runner/cli.rb:167:in` get_args_config_enrichments_resolver’
uri:classloader:/gems/contracts-0.11.0/lib/contracts/method_reference.rb:43:in `send_to' uri:classloader:/gems/contracts-0.11.0/lib/contracts/call_with.rb:76:in` call_with’
uri:classloader:/gems/contracts-0.11.0/lib/contracts/method_handler.rb:138:in `block in redefine_method' uri:classloader:/emr-etl-runner/bin/snowplow-emr-etl-runner:37:in` ’
org/jruby/RubyKernel.java:979:in `load' uri:classloader:/META-INF/main.rb:1:in` ’
org/jruby/RubyKernel.java:961:in `require' uri:classloader:/META-INF/main.rb:1:in` (root)’
uri:classloader:/META-INF/jruby.home/lib/ruby/stdlib/rubygems/core_ext/kernel_require.rb:1:in `’
This is the config yml file:
aws:
access_key_id: #####
secret_access_key: #####
s3:
region: eu-central-1
buckets:
assets: s3://snowplow-hosted-assets
jsonpath_assets:
log: s3://snowplow-etl-emr/logs
raw:
in:
- s3://snowplow-logs-gtm
processing: s3://swapfiets-snowplow-data/processing
archive: s3://swapfiets-snowplow-data/archive/raw
enriched:
good: s3://swapfiets-snowplow-data/enriched/good
bad: s3://swapfiets-snowplow-data/enriched/bad
errors:
archive: s3://swapfiets-snowplow-data/archive/enriched
shredded:
good: s3://swapfiets-snowplow-data/shredded/good
bad: s3://swapfiets-snowplow-data/shredded/bad
errors:
archive: s3://swapfiets-snowplow-data/archive/shredded
emr:
ami_version: 5.9.0
region: eu-central-1
jobflow_role: EMR_EC2_DefaultRole
service_role: EMR_DefaultRole
placement:
ec2_subnet_id: subnet-####
ec2_key_name: ###
bootstrap: []
software:
hbase:
lingual:
jobflow:
job_name: Snowplow ETL
master_instance_type: m1.medium
core_instance_count: 2
core_instance_type: m1.medium
core_instance_ebs:
volume_size: 100
volume_type: "gp2"
volume_iops: 400
ebs_optimized: false
task_instance_count: 0
task_instance_type: m1.medium
task_instance_bid: 0.015
additional_info:
bootstrap_failure_tries: 3
configuration:
collectors:
format: cloudfront
enrich:
versions:
spark_enrich: 1.10.0
continue_on_unexpected_error: false
output_compression: NONE
storage:
versions:
rdb_shredder: 0.13.0
hadoop_elasticsearch: 0.1.0
rdb_loader: 0.14.0
monitoring:
tags: {}
logging:
level: DEBUG
snowplow:
method: get
collector: ####.cloudfront.net
app_id:
Hopefully someone can help me, would be awewome!
Many thanks!
Willem