Hi there,
I haven’t been able to get the ETLEMR process going. I have scoured through other similar issue and can’t find a solution.
**ERROR:**
$ ./snowplow-emr-etl-runner -d --config config/config.yml --resolver resolver.json
F, [2016-06-10T02:54:57.229000 #26357] FATAL -- :
ContractError (Contract violation for return value:
Expected: {:aws=>{:access_key_id=>String, :secret_access_key=>String, :s3=>{:region=>String, :buckets=>{:assets=>String, :jsonpath_assets=>#<Contracts::Maybe:0x122a10c5 @vals=[String, nil]>, :log=>String, :raw=>{:in=>#<Contracts::ArrayOf:0x7b2fed4 @contract=String>, :processing=>String, :archive=>String}, :enriched=>{:good=>String, :bad=>String, :errors=>#<Contracts::Maybe:0x7878143e @vals=[String, nil]>, :archive=>#<Contracts::Maybe:0x1379303c @vals=[String, nil]>}, :shredded=>{:good=>String, :bad=>String, :errors=>#<Contracts::Maybe:0x794dbd20 @vals=[String, nil]>, :archive=>#<Contracts::Maybe:0x5ebbbe17 @vals=[String, nil]>}}}, :emr=>{:ami_version=>String, :region=>String, :jobflow_role=>String, :service_role=>String, :placement=>#<Contracts::Maybe:0x77bd0897 @vals=[String, nil]>, :ec2_subnet_id=>#<Contracts::Maybe:0x1b7f2eeb @vals=[String, nil]>, :ec2_key_name=>String, :bootstrap=>#<Contracts::Maybe:0x3ae15467 @vals=[#<Contracts::ArrayOf:0x1def1f1e @contract=String>, nil]>, :software=>{:hbase=>#<Contracts::Maybe:0x1f6c5464 @vals=[String, nil]>, :lingual=>#<Contracts::Maybe:0x118815a @vals=[String, nil]>}, :jobflow=>{:master_instance_type=>String, :core_instance_count=>Contracts::Num, :core_instance_type=>String, :task_instance_count=>Contracts::Num, :task_instance_type=>String, :task_instance_bid=>#<Contracts::Maybe:0x6fd43c45 @vals=[Contracts::Num, nil]>}, :additional_info=>#<Contracts::Maybe:0x56c10f5e @vals=[String, nil]>, :bootstrap_failure_tries=>Contracts::Num}}, :collectors=>{:format=>String}, :enrich=>{:job_name=>String, :versions=>{:hadoop_enrich=>String, :hadoop_shred=>String}, :continue_on_unexpected_error=>Contracts::Bool, :output_compression=>#<Proc:0x3a9b7e55@/home/ec2-user/snowplow-emr-etl-runner!/emr-etl-runner/lib/snowplow-emr-etl-runner/contracts.rb:23 (lambda)>}, :storage=>{:download=>{:folder=>#<Contracts::Maybe:0x2d381822 @vals=[String, nil]>}, :targets=>#<Contracts::ArrayOf:0x3db8775f @contract={:name=>String, :type=>String, :host=>String, :database=>String, :port=>Contracts::Num, :ssl_mode=>#<Contracts::Maybe:0x4bc348d3 @vals=[String, nil]>, :table=>String, :username=>#<Contracts::Maybe:0x2bb0931e @vals=[String, nil]>, :password=>#<Contracts::Maybe:0x3627cf6 @vals=[String, nil]>, :es_nodes_wan_only=>#<Contracts::Maybe:0x561d596c @vals=[Contracts::Bool, nil]>, :maxerror=>#<Contracts::Maybe:0x437f7292 @vals=[Contracts::Num, nil]>, :comprows=>#<Contracts::Maybe:0x59dcd5ec @vals=[Contracts::Num, nil]>}>}, :monitoring=>{:tags=>#<Contracts::HashOf:0x19a51da1 @value=String, @key=Symbol>, :logging=>{:level=>String}, :snowplow=>#<Contracts::Maybe:0x273c422e @vals=[{:method=>String, :collector=>String, :app_id=>String}, nil]>}},
Actual: {:aws=>{:access_key_id=>"AKIAIUFF5DVIXMMDFVFA", :secret_access_key=>"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", :s3=>{:region=>"ap-southeast-2", :buckets=>{:assets=>"s3://snowplow-hosted-assets", :jsonpath_assets=>nil, :log=>"s3://ETLEMR/ETLEMR_logs", :raw=>{:in=>"s3://elasticbeanstalk-ap-southeast-2-098002129817/resources/environments/logs/publish/e-m3cepm4223/i-7232cba3", :processing=>"s3://ETLEMR/ETLEMR_in_processing", :archive=>"s3://ETLEMR/ETLEMR_in_archive"}, :enriched=>{:good=>"s3://ETLEMR/ETLEMR_enriched_good", :bad=>"s3://ETLEMR/ETLEMR_enriched_bad", :errors=>"s3://ETLEMR/ETLEMR_enriched_errors", :archive=>"s3://ETLEMR/ETLEMR_enriched_archive"}, :shredded=>{:good=>"s3://ETLEMR/ETLEMR_shredded_good", :bad=>"s3://ETLEMR/ETLEMR_shredded_bad", :errors=>"s3://ETLEMR/ETLEMR_shredded_errors", :archive=>"s3://ETLEMR/ETLEMR_shredded_archive"}}}, :emr=>{:ami_version=>"4.5.0", :region=>"ap-southeast-2", :jobflow_role=>"EMR_EC2_DefaultRole", :service_role=>"EMR_DefaultRole", :placement=>nil, :ec2_subnet_id=>nil, :ec2_key_name=>"firstec2.ppk", :bootstrap=>[], :software=>{:hbase=>nil, :lingual=>nil}, :jobflow=>{:master_instance_type=>"m1.medium", :core_instance_count=>2, :core_instance_type=>"m1.medium", :task_instance_count=>0, :task_instance_type=>"m1.medium", :task_instance_bid=>0.015}, :bootstrap_failure_tries=>3, :additional_info=>nil}}, :collectors=>{:format=>"clj-tomcat"}, :enrich=>{:job_name=>"Snowplow ETL", :versions=>{:hadoop_enrich=>"1.7.0", :hadoop_shred=>"0.9.0", :hadoop_elasticsearch=>"0.1.0"}, :continue_on_unexpected_error=>false, :output_compression=>"NONE"}, :storage=>{:download=>{:folder=>nil}, :targets=>[{:name=>"My Redshift database", :type=>"redshift", :host=>"ADD HERE", :database=>"ADD HERE", :port=>5439, :table=>"atomic.events", :username=>"ADD HERE", :password=>"ADD HERE", :maxerror=>1, :comprows=>200000, :ssl_mode=>"disable"}]}, :monitoring=>{:tags=>{}, :logging=>{:level=>"DEBUG"}, :snowplow=>{:method=>"get", :app_id=>"redplanet", :collector=>"ec2-52-63-114-56.ap-southeast-2.compute.amazonaws.com"}}}
Value guarded in: Snowplow::EmrEtlRunner::Cli::load_config
With Contract: Maybe, String => Hash
At: /home/ec2-user/snowplow-emr-etl-runner!/emr-etl-runner/lib/snowplow-emr-etl-runner/cli.rb:134 ):
/home/ec2-user/snowplow-emr-etl-runner!/gems/contracts-0.7/lib/contracts.rb:69:in `Contract'
org/jruby/RubyProc.java:271:in `call'
/home/ec2-user/snowplow-emr-etl-runner!/gems/contracts-0.7/lib/contracts.rb:147:in `failure_callback'
/home/ec2-user/snowplow-emr-etl-runner!/gems/contracts-0.7/lib/contracts/decorators.rb:164:in `common_method_added'
/home/ec2-user/snowplow-emr-etl-runner!/gems/contracts-0.7/lib/contracts/decorators.rb:159:in `common_method_added'
file:/home/ec2-user/snowplow-emr-etl-runner!/emr-etl-runner/bin/snowplow-emr-etl-runner:37:in `(root)'
org/jruby/RubyKernel.java:1091:in `load'
file:/home/ec2-user/snowplow-emr-etl-runner!/META-INF/main.rb:1:in `(root)'
org/jruby/RubyKernel.java:1072:in `require'
file:/home/ec2-user/snowplow-emr-etl-runner!/META-INF/main.rb:1:in `(root)'
/tmp/jruby8204250454434082145extract/jruby-stdlib-1.7.20.1.jar!/META-INF/jruby.home/lib/ruby/shared/rubygems/core_ext/kernel_require.rb:1:in `(root)'
Here is my config.yml:
aws:
# Credentials can be hardcoded or set in environment variables
access_key_id: XXXXXXXXXXXXXXXXXX
secret_access_key: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
s3:
region: ap-southeast-2
buckets:
assets: s3://snowplow-hosted-assets # DO NOT CHANGE unless you are hosting the jarfiles etc yourself in your own bucket
jsonpath_assets: # If you have defined your own JSON Schemas, add the s3:// path to your own JSON Path files in your own bucket here
log: s3://ETLEMR/ETLEMR_logs
raw:
in: s3://elasticbeanstalk-ap-southeast-2-098002129817/resources/environments/logs/publish/e-m3cepm4223/i-7232cba3 # Multiple in buckets are permitted
# e.g. s3://my-in-bucket
processing: s3://ETLEMR/ETLEMR_in_processing
archive: s3://ETLEMR/ETLEMR_in_archive # e.g. s3://my-archive-bucket/in
enriched:
good: s3://ETLEMR/ETLEMR_enriched_good # e.g. s3://my-out-bucket/enriched/good
bad: s3://ETLEMR/ETLEMR_enriched_bad # e.g. s3://my-out-bucket/enriched/bad
errors: s3://ETLEMR/ETLEMR_enriched_errors # Leave blank unless continue_on_unexpected_error: set to true below
archive: s3://ETLEMR/ETLEMR_enriched_archive # Where to archive enriched events to, e.g. s3://my-archive-bucket/enriched
shredded:
good: s3://ETLEMR/ETLEMR_shredded_good # e.g. s3://my-out-bucket/shredded/good
bad: s3://ETLEMR/ETLEMR_shredded_bad # e.g. s3://my-out-bucket/shredded/bad
errors: s3://ETLEMR/ETLEMR_shredded_errors # Leave blank unless continue_on_unexpected_error: set to true below
archive: s3://ETLEMR/ETLEMR_shredded_archive # Where to archive shredded events to, e.g. s3://my-archive-bucket/shredded
emr:
ami_version: 4.5.0 # Don't change this
region: ap-southeast-2 # Always set this
jobflow_role: EMR_EC2_DefaultRole # Created using $ aws emr create-default-roles
service_role: EMR_DefaultRole # Created using $ aws emr create-default-roles
placement: # Set this if not running in VPC. Leave blank otherwise
ec2_subnet_id: # Set this if running in VPC. Leave blank otherwise
ec2_key_name: firstec2.ppk
bootstrap: [] #Set this to specify custom boostrap actions. Leave empty otherwise.
software:
hbase: # Optional. To launch on cluster, provide version, "0.92.0", keep quotes. Leave empty otherwise.
lingual: # Optional. To launch on cluster, provide version, "1.1", keep quotes. Leave empty otherwise.
# Adjust your Hadoop cluster below
jobflow:
master_instance_type: m1.medium
core_instance_count: 2
core_instance_type: m1.medium
task_instance_count: 0 # Increase to use spot instances
task_instance_type: m1.medium
task_instance_bid: 0.015 # In USD. Adjust bid, or leave blank for non-spot-priced (i.e. on-demand) task instances
bootstrap_failure_tries: 3 # Number of times to attempt the job in the event of bootstrap failures
additional_info: # Optional JSON string for selecting additional features
collectors:
format: clj-tomcat # Or 'clj-tomcat' for the Clojure Collector, or 'thrift' for Thrift records, or 'tsv/com.amazon.aws.cloudfront/wd_access_log' for Cloudfront access logs
enrich:
job_name: Snowplow ETL # Give your job a name
versions:
hadoop_enrich: 1.7.0 # Version of the Hadoop Enrichment process
hadoop_shred: 0.9.0 # Version of the Hadoop Shredding process
hadoop_elasticsearch: 0.1.0 # Version of the Hadoop to Elasticsearch copying process
continue_on_unexpected_error: false # Set to 'true' (and set out_errors: above) if you don't want any exceptions thrown from ETL
output_compression: NONE # Compression only supported with Redshift, set to NONE if you have Postgres targets. Allowed formats: NONE, GZIP
storage:
download:
folder: # Postgres-only config option. Where to store the downloaded files. Leave blank for Redshift
targets:
- name: "My Redshift database"
type: redshift
host: ADD HERE # The endpoint as shown in the Redshift console
database: ADD HERE # Name of database
port: 5439 # Default Redshift port
table: atomic.events
username: ADD HERE
password: ADD HERE
maxerror: 1 # Stop loading on first error, or increase to permit more load errors
comprows: 200000 # Default for a 1 XL node cluster. Not used unless --include compupdate specified
ssl_mode: disable
#- name: "My PostgreSQL database"
# type: postgres
# host: ADD HERE # Hostname of database server
# database: ADD HERE # Name of database
# port: 5432 # Default Postgres port
# table: atomic.events
# username: ADD HERE
# password: ADD HERE
# maxerror: # Not required for Postgres
# comprows: # Not required for Postgres
# ssl_mode: disable
#- name: "myelasticsearchtarget" # Name for the target - used to label the corresponding jobflow step
# type: elasticsearch # Marks the database type as Elasticsearch
# host: "ec2-43-1-854-22.compute-1.amazonaws.com" # Elasticsearch host
# database: index1 # The Elasticsearch index
# port: 9200 # Port used to connect to Elasticsearch
# table: type1 # The Elasticsearch type
# es_nodes_wan_only: false # Set to true if using Amazon Elasticsearch Service
# username: # Unnecessary for Elasticsearch
# password: # Unnecessary for Elasticsearch
# sources: # Leave blank or specify: ["s3://out/enriched/bad/run=xxx", "s3://out/shred/bad/run=yyy"]
# maxerror: # Not required for Elasticsearch
# comprows: # Not required for Elasticsearch
monitoring:
tags: {} # Name-value pairs describing this job
logging:
level: DEBUG # You can optionally switch to INFO for production
snowplow:
method: get
app_id: redplanet # e.g. snowplow
collector: ec2-52-XX-XXX-XX.ap-southeast-2.compute.amazonaws.com # e.g. d3rkrsqld9gmqf.cloudfront.net
Your help would be greatly appreciated.
