Config Templates - mvrpl/big-shipper GitHub Wiki

delimitedfile - Template

{
	"SOURCE":{
		"TYPE": "delimitedfile",
		"FIELDS": [
			{
				"NAME": "field1",
				"TYPE": "int"
			},
			{
				"NAME": "field2",
				"TYPE": "string"
			},
			{
				"NAME": "field3",
				"TYPE": "decimal"
			}
		],
		"DELIMITER_RAW": "|",
		"DIR_RAW_FILES": "/user/NAME/data_201703{2[7-9],3[0-1]}.txt"
	},
	"TARGET":{
		"TYPE": "hive",
		"HIVE_TABLE": "table_name_here"
	}
}

json - Template one partition

{
        "SOURCE":{
                "TYPE": "json",
                "SCHEMA": "{\"col1\":\"string\",\"col2\":\"string\",\"col3\":\"string\"}",
                "DIR_RAW_FILES": "file:///home/USER/*.json"
        },
        "TARGET":{
                "TYPE": "hive",
                "HIVE_TABLE": "TABLENAME",
                "PARTITION": "col1"
        }
}

json - Template multi-partition

{
        "SOURCE":{
                "TYPE": "json",
                "SCHEMA": "{\"col1\":\"string\",\"col2\":\"string\",\"col3\":\"string\"}",
                "DIR_RAW_FILES": "file:///home/USER/*.json"
        },
        "TARGET":{
                "TYPE": "hive",
                "HIVE_TABLE": "TABLENAME",
                "PARTITION": ["col1", "col2"]
        }
}