sp3sm - Hari-dta/hari GitHub Wiki

CREATE external TABLE acabes_training.mongo_to_rdv_CONTROL( DB_USER STRING, DB_PASS STRING, DB_HOST STRING, DB_PORT INT, DB_NAME STRING, db STRING, collection STRING, stgname STRING, targetname STRING, audittable STRING, loadstatustable STRING, writetype STRING )stored as parquet

INSERT INTO TABLE acabes_training.mongo_to_rdv_CONTROL ( DB_USER, DB_PASS, DB_HOST, DB_PORT, DB_NAME, db, collection, stgname, targetname, audittable, loadstatustable, writetype ) VALUES ( 'applicationtest', 'DY8IizVeYmmepNzuu9H6aDnakAHKNf', '10.0.53.60', 27028, 'Merchant_Portal', 'Merchant_Portal', 'sample_nested_collection', 'acabes_training.mongo_to_rdv_stg', 'acabes_training.mongo_to_rdv_target', 'acabes_training.mongo_to_rdv_audit_table', 'acabes_training.mongo_to_rdv_loadingstatus', 'overwrite' );

spark3-submit --master yarn --deploy-mode client --executor-memory 20g --num-executors 20 --driver-memory 20g --driver-cores 20 --conf spark.shuffle.compress=true --conf spark.io.compression.codec=snappy --conf spark.streaming.backpressure.enabled=true --conf spark.yarn.maxAppAttempts=2 --conf "spark.executor.extraJavaOptions=-XX:+UseG1GC -Djavax.net.ssl.trustStore=/home/hariharan-m/truststore.jks -Djavax.net.ssl.trustStorePassword=changeit" --conf "spark.driver.extraJavaOptions=-XX:+UseG1GC -Djavax.net.ssl.trustStore=/home/hariharan-m/truststore.jks -Djavax.net.ssl.trustStorePassword=changeit" --jars /home/acabes_dev_team/mongo/mongo-spark-connector_2.12-10.2.1.jar,/home/acabes_dev_team/mongo/bson-4.11.1.jar,/home/acabes_dev_team/mongo/mongodb-driver-sync-4.11.1.jar,/home/acabes_dev_team/mongo/mongodb-driver-core-4.11.1.jar,/home/hariharan-m/mongodb-driver-sync-4.11.1.jar,/home/hariharan-m/hive-serde-4.0.1.jar,/home/hariharan-m/hive-hcatalog-core-0.13.0.jar,/home/hariharan-m/json-se--files /home/hariharan-m/mongodb-shared-chain.pem "/home/hariharan-m/mongo_to_rdv_final/MONGO_TO_RDV_DYNAMIC.py" --uri "mongodb://applicationtest:[email protected]:27028/Merchant_Portal?ssl=true&ssl_ca_certs=/home/hariharan-m/mongodb-shared-chain.pem&authSource=Merchant_Portal&readPreference=primary&socketTimeoutMS=30000" --db "Merchant_Portal" --collection "BkMerchantPortalJO_2_6" --stgname "acabes_training.mongo_to_rdv_stg" --targetname "acabes_training.mongo_to_rdv_target" --audittable "acabes_training.mongo_to_rdv_audit_table" --loadstatustable "acabes_training.mongo_to_rdv_loadingstatus" --writetype "overwrite"