Kinesis - aidarko/dev-notes GitHub Wiki

Get message

=======================
==== GET MESSAGE   ====
=======================
SHARD_ITERATOR=$(aws kinesis --profile kinesis-test --region=eu-west-1 get-shard-iterator --shard-id shardId-000000000000 --shard-iterator-type TRIM_HORIZON --stream-name stream-name-ingest --query 'ShardIterator' --no-verify-ssl | tr -d \")
AWS_RECORD=$(aws --profile kinesis-test --region=eu-west-1 kinesis get-records --shard-iterator ${SHARD_ITERATOR} --no-verify-ssl)
echo ${AWS_RECORD} | grep Data
   
{"date":"value-248-2019-11-20T09:59:58.747Z","randomNumber":00000,"timestamp":"000"}

Put message

=======================
==== PUT MESSAGE   ====
=======================
aws --profile kinesis-test --region=eu-west-1 kinesis \
put-record \
--data "{test-yo}" \
--stream-name stream-name-ingest \
--partition-key whatever   

Iterate shard

=======================
==== ITERATE SHARD ====
=======================
# encrypted 
SHARD_ITERATOR=$(aws kinesis --profile kinesis-test --region=eu-west-1 get-shard-iterator --shard-id shardId-000000000000 --shard-iterator-type TRIM_HORIZON --stream-name stream-name-ingest --query 'ShardIterator' | tr -d \")
while [ true ]
do
    AWS_RECORD=$(aws --profile kinesis-test --region=eu-west-1 kinesis get-records --shard-iterator ${SHARD_ITERATOR})
    DATA=$(echo ${AWS_RECORD} | grep Data)
        echo ${DATA} >> data_stream.txt
        echo "" >> data_stream.txt
    SHARD_ITERATOR=$(echo ${AWS_RECORD} | jq -r '.NextShardIterator')
done

# decrypted
SHARD_ITERATOR=$(aws kinesis --profile kinesis-test --region=eu-west-1 get-shard-iterator --shard-id shardId-000000000000 --shard-iterator-type TRIM_HORIZON --stream-name stream-name-ingest --query 'ShardIterator' | tr -d \")
while [ true ]
do
    AWS_RECORD=$(aws --profile kinesis-test --region=eu-west-1 kinesis get-records --shard-iterator ${SHARD_ITERATOR})
    DATA=$(echo ${AWS_RECORD} | grep Data)
        echo ${DATA} | while read line ; do
             echo "$line" | sed 's/"Data": "//g' | sed 's/",//g' | xargs | base64 --decode >> data_stream.txt
             echo "" >> data_stream.txt
        done
    SHARD_ITERATOR=$(echo ${AWS_RECORD} | jq -r '.NextShardIterator')
done

with time

#set time
    FROM_TIMESTAMP=$(date -v -24H +%Y-%m-%dT%H:%M:%S.400-00:00)
# list shards
    aws kinesis --profile test --region=eu-west-1 list-shards --stream-name stream-name | grep \"ShardId\":
# set shard iterator
    SHARD_ITERATOR=$(aws kinesis --profile test --region=eu-west-1 get-shard-iterator --shard-id shardId-000000000002 --shard-iterator-type AT_TIMESTAMP --timestamp ${FROM_TIMESTAMP} --stream-name stream-name --query 'ShardIterator' | tr -d \")

# write to file data_stream.txt
while [ true ]
do
    AWS_RECORD=$(aws --profile test --region=eu-west-1 kinesis get-records --shard-iterator ${SHARD_ITERATOR})
    DATA=$(echo ${AWS_RECORD} | grep Data)
        echo ${DATA} | while read line ; do
             echo "$line" | sed 's/"Data": "//g' | sed 's/",//g' | xargs | base64 --decode >> data_stream2.txt
             echo "" >> data_stream.txt
        done
    SHARD_ITERATOR=$(echo ${AWS_RECORD} | jq -r '.NextShardIterator')
done