Hi Guys, My snowplow-emr-etl-runner madara_rider fails while loading data into redshift. with the error mentioned in title of this post.
Below is my rdb_targets config json file:
{
“schema”: “iglu:com.snowplowanalytics.snowplow.storage/redshift_config/jsonschema/3-0-0”,
“data”: {
“name”: “Snowplow AWS Redshift enriched events storage”,
“id”: “xxxxxxxx-xxxx-xxxx-xxxx-13fa8efc57c7”,
“host”: “xxxxx.us-east-1.redshift.amazonaws.com”,
“database”: “snowplowdb”,
“port”: 5439,
“username”: “storageloader”,
“password”: “xxxxxxx”,
“roleArn”: “arn:aws:iam::############:role/SnowplowRedshiftLoadRole”,
“schema”: “atomic”,
“maxError”: 1,
“compRows”: 5000,
“sshTunnel”: null,
“processingManifest”: null,
“jdbc”: {
“ssl”: false
},
“purpose”: “ENRICHED_EVENTS”
}
}
Below is my emr-etl-runner config.yaml
aws:
# Credentials can be hardcoded or set in environment variables
access_key_id: <%= ENV['AWS_SNOWPLOW_ACCESS_KEY'] %>
secret_access_key: <%= ENV['AWS_SNOWPLOW_SECRET_KEY'] %>
s3:
region: us-east-1
buckets:
assets: s3://snowplow-hosted-assets # DO NOT CHANGE unless you are hosting the jarfiles etc yourself in your own bucket
jsonpath_assets: # If you have defined your own JSON Schemas, add the s3:// path to your own JSON Path files in your own bucket here
log: s3://xxxxxxxxx-snowplow-emr-etl-runner/logs
encrypted: false
enriched:
good: s3://xxxxxxxxx-snowplow-emr-etl-runner/enriched/good # e.g. s3://my-out-bucket/enriched/good
archive: s3://xxxxxxxxx-snowplow-emr-etl-runner/enriched/archive # Where to archive enriched events to, e.g. s3://my-archive-bucket/enriched
stream: s3://xxxxxxxxx-snowplow-s3-loader # S3 Loader's output folder with enriched data. If present raw buckets will be discarded
shredded:
good: s3://xxxxxxxxx-snowplow-emr-etl-runner/shredded/good # e.g. s3://my-out-bucket/shredded/good
bad: s3://xxxxxxxxx-snowplow-emr-etl-runner/shredded/bad # e.g. s3://my-out-bucket/shredded/bad
errors: # s3://xxxxxxxxx-snowplow-emr-etl-runner/shredded/errors # Leave blank unless :continue_on_unexpected_error: set to true below
archive: s3://xxxxxxxxx-snowplow-emr-etl-runner/shredded/archive # Where to archive shredded events to, e.g. s3://my-archive-bucket/shredded
consolidate_shredded_output: false # Whether to combine files when copying from hdfs to s3
emr:
ami_version: 5.9.0
region: us-east-1 # Always set this
jobflow_role: EMR_EC2_DefaultRole # Created using $ aws emr create-default-roles
service_role: EMR_DefaultRole # Created using $ aws emr create-default-roles
placement: us-east-1c # Set this if not running in VPC. Leave blank otherwise
ec2_subnet_id: # Set this if running in VPC. Leave blank otherwise
ec2_key_name: snowplow
security_configuration: # Specify your EMR security configuration if needed. Leave blank otherwise
bootstrap: [] # Set this to specify custom boostrap actions. Leave empty otherwise
software:
hbase: # Optional. To launch on cluster, provide version, "0.92.0", keep quotes. Leave empty otherwise.
lingual: # Optional. To launch on cluster, provide version, "1.1", keep quotes. Leave empty otherwise.
# Adjust your Hadoop cluster below
jobflow:
job_name: xxxxxxxxx Snowplow Stream ETL # Give your job a name
master_instance_type: m4.large
core_instance_count: 1
core_instance_type: m4.large
core_instance_ebs: # Optional. Attach an EBS volume to each core instance.
volume_size: 100 # Gigabytes
volume_type: "gp2"
volume_iops: 400 # Optional. Will only be used if volume_type is "io1"
ebs_optimized: false # Optional. Will default to true
task_instance_count: 2 # Increase to use spot instances
task_instance_type: m4.large
task_instance_bid: 0.033 # In USD. Adjust bid, or leave blank for non-spot-priced (i.e. on-demand) task instances
bootstrap_failure_tries: 3 # Number of times to attempt the job in the event of bootstrap failures
configuration:
yarn-site:
yarn.resourcemanager.am.max-attempts: "1"
spark:
maximizeResourceAllocation: "true"
additional_info: # Optional JSON string for selecting additional features
enrich:
versions:
spark_enrich: 1.18.0 # Version of the Spark Enrichment process
output_compression: GZIP # Stream mode supports only GZIP
storage:
versions:
rdb_loader: 0.14.0
rdb_shredder: 0.13.1 # Version of the Spark Shredding process
hadoop_elasticsearch: 0.1.0 # Version of the Hadoop to Elasticsearch copying process
monitoring:
tags: {} # Name-value pairs describing this job
logging:
level: DEBUG # You can optionally switch to INFO for production
snowplow:
method: post
app_id: snowplow-xxxxxx-emr-etl-runner # e.g. snowplow
collector: spcollector.xxxxxxxxx.com # e.g. dddddd9gmqf.cloudfront.net
protocol: https
port: 443
Job:
Below is the controller log output of the failed step:
2019-11-04T18:23:50.826Z INFO Ensure step 7 jar file s3://snowplow-hosted-assets-us-east-1/4-storage/rdb-loader/snowplow-rdb-loader-0.14.0.jar
2019-11-04T18:23:52.325Z INFO StepRunner: Created Runner for step 7
INFO startExec ‘hadoop jar /mnt/var/lib/hadoop/steps/s-33XR9FU84CJQ4/snowplow-rdb-loader-0.14.0.jar --config LS0tCmF3c…//A0NDMK --resolver ewogI…9Cn0K --logkey s3://xxxxxxxx-snowplow-emr-etl-runner/logs/rdb-loader/2019-11-04-18-14-30/xxxxxxxx-xxxx-4f33-xxxx-a031d7a36d3f --target eyJzY…ifX0=’
INFO Environment:
PATH=/sbin:/usr/sbin:/bin:/usr/bin:/usr/local/sbin:/opt/aws/bin
LESS_TERMCAP_md=[01;38;5;208m
LESS_TERMCAP_me=[0m
HISTCONTROL=ignoredups
LESS_TERMCAP_mb=[01;31m
AWS_AUTO_SCALING_HOME=/opt/aws/apitools/as
UPSTART_JOB=rc
LESS_TERMCAP_se=[0m
HISTSIZE=1000
HADOOP_ROOT_LOGGER=INFO,DRFA
JAVA_HOME=/etc/alternatives/jre
AWS_DEFAULT_REGION=us-east-1
AWS_ELB_HOME=/opt/aws/apitools/elb
LESS_TERMCAP_us=[04;38;5;111m
EC2_HOME=/opt/aws/apitools/ec2
TERM=linux
XFILESEARCHPATH=/usr/dt/app-defaults/%L/Dt
runlevel=3
LANG=en_US.UTF-8
AWS_CLOUDWATCH_HOME=/opt/aws/apitools/mon
MAIL=/var/spool/mail/hadoop
LESS_TERMCAP_ue=[0m
LOGNAME=hadoop
PWD=/
LANGSH_SOURCED=1
HADOOP_CLIENT_OPTS=-Djava.io.tmpdir=/mnt/var/lib/hadoop/steps/s-33XR9FU84CJQ4/tmp
_=/etc/alternatives/jre/bin/java
CONSOLETYPE=serial
RUNLEVEL=3
LESSOPEN=||/usr/bin/lesspipe.sh %s
previous=N
UPSTART_EVENTS=runlevel
AWS_PATH=/opt/aws
USER=hadoop
UPSTART_INSTANCE=
PREVLEVEL=N
HADOOP_LOGFILE=syslog
PYTHON_INSTALL_LAYOUT=amzn
HOSTNAME=ip-172-31-44-134
NLSPATH=/usr/dt/lib/nls/msg/%L/%N.cat
HADOOP_LOG_DIR=/mnt/var/log/hadoop/steps/s-33XR9FU84CJQ4
EC2_AMITOOL_HOME=/opt/aws/amitools/ec2
SHLVL=5
HOME=/home/hadoop
HADOOP_IDENT_STRING=hadoop
INFO redirectOutput to /mnt/var/log/hadoop/steps/s-33XR9FU84CJQ4/stdout
INFO redirectError to /mnt/var/log/hadoop/steps/s-33XR9FU84CJQ4/stderr
INFO Working dir /mnt/var/lib/hadoop/steps/s-33XR9FU84CJQ4
INFO ProcessRunner started child process 10049 :
hadoop 10049 4052 0 18:23 ? 00:00:00 /bin/bash /usr/bin/hadoop jar /mnt/var/lib/hadoop/steps/s-33XR9FU84CJQ4/snowplow-rdb-loader-0.14.0.jar --config LS0tCmF3czoKICBhY2Nlc3Nfa2V5…dG9jb2w6IGh0dHBzCiAgICBwb3J0OiA0NDMK --resolver ewogICJzY2hl…Cn0K --logkey s3://xxxxxxxx-snowplow-emr-etl-runner/logs/rdb-loader/2019-11-04-18-14-30/ba01d33c-114a-4f33-8841-a031d7a36d3f --target eyJzY2hlbWEiOiJpZ…FTlJJQ0hFRF9FVkVOVFMifX0=
2019-11-04T18:23:52.361Z INFO HadoopJarStepRunner.Runner: startRun() called for s-33XR9FU84CJQ4 Child Pid: 10049
INFO Synchronously wait child process to complete : hadoop jar /mnt/var/lib/hadoop/steps/s-33XR9FU8…
INFO waitProcessCompletion ended with exit code 1 : hadoop jar /mnt/var/lib/hadoop/steps/s-33XR9FU8…
INFO total process run time: 12 seconds
2019-11-04T18:24:04.463Z INFO Step created jobs:
2019-11-04T18:24:04.463Z WARN Step failed with exitCode 1 and took 12 seconds
Thanks and Regards,
Jasmeet
