Apache Pig Script
Pig
x
26
1
SENSEX.pig2
3
A = LOAD '/hdfs/bhavesh/SENSEX/OUTPUT/HighDemandMarket-r-00000' using PigStorage('\t') as (Sid:int,Sname:chararray,Ttrading:chararray,Sloc:chararray,OBal:int,CBal:int,Frate:int);4
disHM = DISTINCT A;5
orHM = ORDER disHM by Sid;6
STORE orHM into '/hdfs/bhavesh/SENSEX/HM' using PigStorage(',');7
8
A = LOAD '/hdfs/bhavesh/SENSEX/OUTPUT/ReliableProducts-r-00000' using PigStorage('\t') as (Sid:int,Sname:chararray,Ttrading:chararray,Sloc:chararray,OBal:int,CBal:int,Frate:int);9
disRP = DISTINCT A;10
orRP = ORDER disRP by Sid;11
STORE orRP into '/hdfs/bhavesh/SENSEX/RP' using PigStorage(',');12
13
A = LOAD '/hdfs/bhavesh/SENSEX/OUTPUT/OtherProducts-r-00000' using PigStorage('\t') as (Sid:int,Sname:chararray,Ttrading:chararray,Sloc:chararray,OBal:int,CBal:int,Frate:int);14
disOP = DISTINCT A;15
orOP = ORDER disOP by Sid;16
STORE orOP into '/hdfs/bhavesh/SENSEX/OP' using PigStorage(',');17
18
A = LOAD '/hdfs/bhavesh/SENSEX/OUTPUT/WealthyProducts-r-00000' using PigStorage('\t') as (Sid:int,Sname:chararray,Ttrading:chararray,Sloc:chararray,OBal:int,CBal:int,Frate:int);19
disWP= DISTINCT A;20
orWP = ORDER disWP by Sid;21
STORE orWP into '/hdfs/bhavesh/SENSEX/WP' using PigStorage(',');22
23
A = LOAD '/hdfs/bhavesh/SENSEX/OUTPUT/OnGoingMarketStretegy-r-00000' using PigStorage('\t') as (Sid:int,Sname:chararray,Ttrading:chararray,Sloc:chararray,OBal:int,CBal:int,Frate:int);24
disOMS = DISTINCT A;25
orOMS = ORDER disOMS by Sid;26
STORE orOMS into '/hdfs/bhavesh/SENSEX/OMS' using PigStorage(',');Shell Script (SENSEX.sh)
Shell
1
109
1
###############################################################################2
############################# COMPLETE SCRIPT ##############################3
### HEADER - PROGRAM NAME - <SENSEX.sh> 4
### AUTHOR - BHAVESH BHADRICHA 5
### DATE - 27/DEC/2015 6
### VERSION - 1.0 7
### DESCRIPTION - Data: Sensex Log Data Processing 8
### (PDF File Processing in Map Reduce) 9
###############################################################################10
###############################################################################11
##################################12
###DEFINING THE LOCAL VARIABLES###13
##################################14
DATE=$(date +"%Y%m%d_%H%M%S")15
LOGFILE="/home/bhavesh/POC/SENSEX/LOG/"$DATE".log"16
17
####### Removing if any existent directories ##################################18
19
hadoop fs -rmr /hdfs/bhavesh/SENSEX/RP20
hadoop fs -rmr /hdfs/bhavesh/SENSEX/WP21
hadoop fs -rmr /hdfs/bhavesh/SENSEX/OP22
hadoop fs -rmr /hdfs/bhavesh/SENSEX/OMS23
hadoop fs -rmr /hdfs/bhavesh/SENSEX/HM24
25
##################################################################################26
############## PDF File Processing USING Map Reduce ##############################27
##################################################################################28
echo "Mapreduce Program starts here"29
30
echo "PDF File Processing in Map Reduce Started" >> $LOGFILE31
32
hadoop fs -rmr /hdfs/bhavesh/SENSEX/OUTPUT33
34
hadoop jar /home/bhavesh/POC/SENSEX/Mapreduce/SENSEX.jar com.bhavesh.poc.sensex.PdfInputDriver /hdfs/bhavesh/SENSEX/INPUT/sensexinputfile.pdf /hdfs/bhavesh/SENSEX/OUTPUT35
36
if [ $? -eq 0 ]; then37
echo "Succesfully finished Mapreduce Processing " >> $LOGFILE38
else39
echo "SENSEX MapReduce Failed Please check the Log " >> $LOGFILE40
fi41
42
#################################################################################43
############### PIG Processing for SEXSEX DATA #################################44
#################################################################################45
46
echo "SENSEX Pig Processing started "47
48
echo "SENSEX PIG Processing Started" >> $LOGFILE49
50
pig -f /home/bhavesh/POC/SENSEX/PIG/SENSEX.pig51
52
if [ $? -eq 0 ]; then53
echo "PIG Succesfully finished SENSEX Processing " >> $LOGFILE54
else55
echo "PIG SENSEX Processing Failed Please check the Log " >> $LOGFILE56
fi57
58
################################################################################59
############# IMPORTING DATA in SQOOP ##########################################60
################################################################################61
62
echo "Importing the data to MYSQL using SQOOP ";63
64
echo "Importing the data to MYSQL " >> $LOGFILE65
66
##### Creating the tables in MySql67
sqoop eval --connect jdbc:mysql://localhost/SENSEX -username root -password root --query "create database if not exists SENSEX;";68
sqoop eval --connect jdbc:mysql://localhost/SENSEX -username root -password root --query "use SENSEX;";69
sqoop eval --connect jdbc:mysql://localhost/SENSEX -username root -password root --query "grant all privileges on SENSEX.* to '%'@'localhost'";70
sqoop eval --connect jdbc:mysql://localhost/SENSEX -username root -password root --query "grant all privileges on SENSEX.* to ''@'localhost'";71
72
sqoop eval --connect jdbc:mysql://localhost/SENSEX -username root -password root --query "drop table if exists HighDemandMarket";73
sqoop eval --connect jdbc:mysql://localhost/SENSEX -username root -password root --query "drop table if exists WealthyProducts";74
sqoop eval --connect jdbc:mysql://localhost/SENSEX -username root -password root --query "drop table if exists OngoingMarketSt";75
sqoop eval --connect jdbc:mysql://localhost/SENSEX -username root -password root --query "drop table if exists ReliableProducts";76
sqoop eval --connect jdbc:mysql://localhost/SENSEX -username root -password root --query "drop table if exists OtherProducts";77
78
echo " MYSQL table creation"79
80
sqoop eval --connect jdbc:mysql://localhost/SENSEX -username root -password root --query "create table HighDemandMarket (Sid int,Sname varchar(30),TType varchar(20),TLoc varchar(20),OpenBal int,CloseBal int,FlucRate int)";81
sqoop eval --connect jdbc:mysql://localhost/SENSEX -username root -password root --query "create table WealthyProducts(Sid int,Sname varchar(30),TType varchar(20),TLoc varchar(20),OpenBal int,CloseBal int,FlucRate int)";82
sqoop eval --connect jdbc:mysql://localhost/SENSEX -username root -password root --query "create table OngoingMarketSt(Sid int,Sname varchar(30),TType varchar(20),TLoc varchar(20),OpenBal int,CloseBal int,FlucRate int)";83
sqoop eval --connect jdbc:mysql://localhost/SENSEX -username root -password root --query "create table ReliableProducts(Sid int,Sname varchar(30),TType varchar(20),TLoc varchar(20),OpenBal int,CloseBal int,FlucRate int)";84
sqoop eval --connect jdbc:mysql://localhost/SENSEX -username root -password root --query "create table OtherProducts(Sid int,Sname varchar(30),TType varchar(20),TLoc varchar(20),OpenBal int,CloseBal int,FlucRate int)";85
86
echo "data exporting";87
88
#### exporting the data into MYSQL89
sqoop export --connect jdbc:mysql://localhost/SENSEX -username root -password root --table HighDemandMarket --export-dir /hdfs/bhavesh/SENSEX/HM/part-r-00000 --fields-terminated-by ',';90
sqoop export --connect jdbc:mysql://localhost/SENSEX -username root -password root --table WealthyProducts --export-dir /hdfs/bhavesh/SENSEX/WP/part-r-00000 --fields-terminated-by ',';91
sqoop export --connect jdbc:mysql://localhost/SENSEX -username root -password root --table OngoingMarketSt --export-dir /hdfs/bhavesh/SENSEX/OMS/part-r-00000 --fields-terminated-by ',';92
sqoop export --connect jdbc:mysql://localhost/SENSEX -username root -password root --table ReliableProducts --export-dir /hdfs/bhavesh/SENSEX/RP/part-r-00000 --fields-terminated-by ',';93
sqoop export --connect jdbc:mysql://localhost/SENSEX -username root -password root --table OtherProducts --export-dir /hdfs/bhavesh/SENSEX/OP/part-r-00000 --fields-terminated-by ',';94
95
if[$? -eq 0]96
echo "exporting of data to MYSQL is done";97
98
echo "exporting of data to MYSQL is done" >> $LOGFILE99
100
echo "creation of hive tables started";101
102
echo "creation of hive tables started " >> $LOGFILE103
104
hive -f /home/bhavesh/POC/SENSEX/HIVE/SENSEX.hql105
106
echo "Hive process is done";107
echo "HIVE PROCESSING is done" >> $LOGFILE108
exit;109
Apache Hive (SENSEX.hql)
Shell
1
42
1
use SENSEX;2
3
Drop table HDM;4
Drop table WP;5
Drop table RP;6
Drop table OP;7
Drop table OMS;8
9
create table HDM(Sid int,Sname string,TTrading string,Sloc String,OpenBal int,CloseBal int,FlucRate int)10
row format delimited11
fields terminated by ","12
stored as textfile;13
14
load data inpath '/hdfs/bhavesh/SENSEX/HM/part-r-00000' into table HDM;15
16
create table WP(Sid int,Sname string,TTrading string,Sloc String,OpenBal int,CloseBal int,FlucRate int)17
row format delimited18
fields terminated by ","19
stored as textfile;20
21
load data inpath '/hdfs/bhavesh/SENSEX/WP/part-r-00000' into table WP;22
23
create table RP(Sid int,Sname string,TTrading string,Sloc String,OpenBal int,CloseBal int,FlucRate int)24
row format delimited25
fields terminated by ","26
stored as textfile;27
28
load data inpath '/hdfs/bhavesh/SENSEX/RP/part-r-00000' into table RP;29
30
create table OP(Sid int,Sname string,TTrading string,Sloc String,OpenBal int,CloseBal int,FlucRate int)31
row format delimited32
fields terminated by ","33
stored as textfile;34
35
load data inpath '/hdfs/bhavesh/SENSEX/OP/part-r-00000' into table OP;36
37
create table OMS(Sid int,Sname string,TTrading string,Sloc String,OpenBal int,CloseBal int,FlucRate int)38
row format delimited39
fields terminated by ","40
stored as textfile;41
42
load data inpath '/hdfs/bhavesh/SENSEX/OMS/part-r-00000' into table OMS;Project Execution
Shell Script Run
Mapreduce Run
Apache Pig Run
Apache Sqoop Run
Apache Hive Run
Mapreduce Output
MYSQL Output
Apache Hive Output
