# bulkload.sh - v1.12

# Directory Locations and other variable settings
ERRORS=0
OIDPROCESS=0
LDAP_DIR=${ORACLE_HOME}/ldap
LDAP_ADMIN=${LDAP_DIR}/admin
LDAP_BIN=${LDAP_DIR}/bin
LDAP_LOG=${LDAP_DIR}/log
LDAP_LOAD=${LDAP_DIR}/load
TMPDIR=${LDAP_DIR}/load
INSTALL_LOG=${LDAP_LOG}/install.log
SCHEMACHECK_LOG=${LDAP_LOG}/schemacheck.log
DATACHECK_LOG=${LDAP_LOG}/duplicateDN.log
NEWNODE_LOG=${LDAP_LOG}/restore.log
#LDAP_CONNECT=onldap
LDAP_CONNECT=''
PSWD="A"
OPTION="n"
bo=``
no=``

# usage message
usage() {
  cat <<BULKLOAD_SH_USAGE
usage: bulkload.sh -connect <connect descriptor> <[-check] [-generate] [-restore] [-load]> <absolute path to LDIF data file>

   -connect  specifies the Net8 connect descriptor to use to connect to
              the database 
   -check    performs LDAP schema and duplicate DN checks on the data file
   -generate creates the Internet Directory Bulk Loader data files for loading
   -restore  Used when ldif file contains Operational Attributes i.e generated
             by ldifwrite. (eg. required when adding new node to an existing 
             replication group.)
   -load     loads the result of a previous -generate phase into the specified
              database
   -encode   specifies the native character set (e.g ".ZHS16GBK" )

   At least one of -check, -generate, or -load actions must be specified.
   The -restore flag should only be used when ldif file contains operational
   attributes such as orclguid, creatorsname etc.
   The pathname to the LDIF data file should be fully specified, and the 
   data file must be specified for the -check or -generate actions.

BULKLOAD_SH_USAGE
  exit 2
}

usage_nofile() {
  cat << BULKLOAD_SH_USAGE_NOFILE
No LDIF data file specified.

BULKLOAD_SH_USAGE_NOFILE
}

usage_noact() {
  cat <<BULKLOAD_SH_USAGE_NOOPT
No action specified.
At least one of -check, -generate, or -load actions must be specified.

BULKLOAD_SH_USAGE_NOOPT
}

usage_nocon() {
  cat <<BULKLOAD_SH_USAGE_NOCON
Unable to detect database using connect descriptor ${LDAP_CONNECT}
Check Net8 client connect descriptor configuration settings.

BULKLOAD_SH_USAGE_NOCON
}

usage_noreadfile() {
  cat <<BULKLOAD_SH_USAGE_NOREADFILE
LDIF Data file '"${BULKSRC}"' not found or unreadable.
check the file and pathname and rerun this script.
(bulkload.sh data file must be specified using fully qualified path)

BULKLOAD_SH_USAGE_NOREADFILE
}

usage_connect() {
  cat << BULKLOAD_SH_USAGE_NOCONNECTSTRING

Specify a valid Net8 connect descriptor to use to connect to the database.

BULKLOAD_SH_USAGE_NOCONNECTSTRING
}

# Executing sqlldr in sequential/parallel mode
direct_load() {

  echo Loading Attribute Store..
  $ORACLE_HOME/bin/sqlldr userid=ods/${PSWD}@$LDAP_CONNECT control=$LDAP_LOAD/attr_store.ctl log=$LDAP_LOG/attr_store.log bad=$LDAP_LOG/attr_store.bad errors=10000000 direct=true 2>&1 >${TMPDIR}/null 

  echo Loading Distinguished Name Catalog..
  $ORACLE_HOME/bin/sqlldr userid=ods/$PSWD@$LDAP_CONNECT control=$LDAP_LOAD/dn.ctl log=$LDAP_LOG/dn.log bad=$LDAP_LOG/dn.bad errors=10000000 direct=true 2>&1 >${TMPDIR}/null 

  echo Loading Binary Attributes..
  $ORACLE_HOME/bin/sqlldr userid=ods/$PSWD@${LDAP_CONNECT} control=$LDAP_LOAD/battrstore.ctl log=$LDAP_LOG/battrstore.log bad=$LDAP_LOG/battrstore.bad errors=10000000 2>&1 >${TMPDIR}/null
  #
  # Optional Search Catalogs
  #
  echo Loading Attribute Search Catalogs..

  # Generate list of sqlldr control files to load for attribute search catalogs
  ctllist=`cd ${LDAP_LOAD} ; ls *.ctl`

  for ctlfile in $ctllist ; do
    # extract attribute name from control file name
    attr=`echo ${ctlfile}|sed -e 's/.ctl//'` ;   
    if [ -z "$attr" -o $attr = "attr_store" -o $attr = "dn" -o $attr = "battrstore" ] ; then
      continue;
    fi
    echo "  " $attr..
    # generate/execute command
    $ORACLE_HOME/bin/sqlldr userid=ods/$PSWD@$LDAP_CONNECT control=$LDAP_LOAD/$attr.ctl log=$LDAP_LOG/$attr.log bad=$LDAP_LOG/$attr.bad errors=10000000 direct=true >${TMPDIR}/null 2>&1
  done
}

# default parameter choices (check no, generate no, load no, 
#                             no default load file)
DOCHECK=0
DOGENERATE=0
DOLOAD=0
BULKSRC=''
NODEFLAG=0
ENCODING=".UTF8"
PARALLEL=0

# parse arguments 
while [ $# -gt 0 ] ; do
  case $1 in
    # -check option disables data file checking
    -check) DOCHECK=1 ;;
    -load) DOLOAD=1 ;;
    -generate) DOGENERATE=1 ;;
    -encode) ENCODING=$2;
             if [ -z "${ENCODING}" ] ; then
               ENCODING=".UTF8";
             fi;
             shift;;
    -restore) NODEFLAG=1 ;;
    -parallel) PARALLEL=1 ;;
    -connect) LDAP_CONNECT=$2 ; 
       if [ -z "${LDAP_CONNECT}" ] ; then
         usage ; 
       fi
       shift ;;
    # Switch to enter odspswd at command line
    -w) PSWD=$2 ; 
        shift ;;
    # anything else must be the data file name
    *) if [ -f $1 -a -z "${BULKSRC}" ] ; then
         BULKSRC=$1
       else
         usage ; 
       fi ;;
  esac
  shift
done

# connect string is mandatory since there is no fool proof way of 
# assuming a default connect string.  $ORACLE_SID is a good default to
# use but cannot always be true.

if [ -z "${LDAP_CONNECT}" ];then
	usage_connect
	usage
fi

# at least one action must be chosen
if [ ${DOLOAD} -eq 0 -a ${DOCHECK} -eq 0 -a ${DOGENERATE} -eq 0 ] ; then
        usage_noact
        usage
fi
# data file must be specified for check or generate actions
if [ -z "${BULKSRC}" -a \( ${DOCHECK} -ne 0 -o ${DOGENERATE} -ne 0 \) ] ; then
        usage_nofile
        usage
fi

# check for required envrionment variable settings
if [ -z "$ORACLE_HOME" ]
then
        echo "ORACLE_HOME must be set before running this script"
        exit 1
fi

##
# check for connectivity to specified connect string
tnsping ${LDAP_CONNECT} >${TMPDIR}/null 2>&1
if [ $? != 0 ] ; then 
  usage_nocon
  usage
fi

# check for supporting SQL files
if [ ! -f ${LDAP_ADMIN}/ldapbnsq.sql ] ; then
        echo "Unable to locate required LDAP support files.  Check installation."
        exit 1
fi
####THis is the Security Implementation######
if [ ${PSWD} = "A" ] ; then
   echo " "
   echo "${bo}This tool can only be executed if you know database user password for OiD${no}"  
   echo " "
   printf "Enter OiD password ::" 
   stty -echo ; read PSWD ; stty echo ; echo
   #clear
fi
echo " set feedback off
   connect ods/${PSWD}@${LDAP_CONNECT} ;
   exit; " > ${TMPDIR}/seccheck.sql
### check if the password provided is correct
${ORACLE_HOME}/bin/sqlplus  odscommon/odscommon@${LDAP_CONNECT} @${TMPDIR}/seccheck.sql  > ${LDAP_LOG}/catchecK.lst
err=`grep -c ERROR ${LDAP_LOG}/catchecK.lst`
rm -f ${LDAP_LOG}/seccheck.lst ${TMPDIR}/seccheck.sql > ${TMPDIR}/null
if [ ${err} != 0 ] ; then
  echo " Password for OiD user is WRONG.."
  echo " Cannot execute this tool ..."
  exit
fi

if [ ${DOLOAD} -eq 1 -o ${DOGENERATE} -eq 1 ] ; then
# See if -check option was not given
 if [ ${DOCHECK} -eq 0 ] ; then
    echo "It is recommended to use -check option before generating/loading data"
    echo "Do you want to continue (y/n)? [n]"

    read OPTION;
    
    #if [ ${OPTION} -neq "y" ];then
    if [ -n "${OPTION}" ]; then
       if [ ${OPTION} != "y" ];then
          exit 1;
       fi
    else exit 1; 
    fi

 fi

 if  [ 0 -eq 1 ]; then
   # verify that no LDAP servers are running...
   PROCESS=`${LDAP_BIN}/ldapcheck oidldapd`
   case $PROCESS in
        *'oidldapd is Alive'*)
                        echo "The Following Processes are still running:"
                        echo "${PROCESS}"
                        echo
                        echo "Shutdown these Processes and start again"
                        exit 1
                        ;;
        *)              ;;
   esac

   # verify that no Replication servers are running...
   PROCESS=`${LDAP_BIN}/ldapcheck oidrepld`
   case $PROCESS in
        *'oidrepld is Alive'*)
                        echo "The Following Processes are still running:"
                        echo "${PROCESS}"
                        echo
                        echo "Shutdown these Processes and start again"
                        exit 1
                        ;;
        *)              ;;
   esac

   # verify that no Guardian servers are running...
   PROCESS=`${LDAP_BIN}/ldapcheck oidmon`
   case $PROCESS in
        *'oidmon is Alive'*)
                        echo "The Following Processes are still running:"
                        echo "${PROCESS}"
                        echo
                        echo "Shutdown these Processes and start again"
                        exit 1
                        ;;
        *)              ;;
   esac
 fi
 $ORACLE_HOME/bin/sqlplus ods/${PSWD}@${LDAP_CONNECT} @${LDAP_ADMIN}/ldapods_process.sql >>${INSTALL_LOG} 2>&1
  OIDPROCESS=`head -1 ${TMPDIR}/dncnt.val | sed -e 's* **g'`
  rm -f /tmp/dncnt.val
  if [ ${OIDPROCESS} != 0 ]; then
     echo "OID Processes running on target node $LDAP_CONNECT"
     echo "Please shutdown OID Processes for bulkload"
     echo
     exit 1
  fi
  $ORACLE_HOME/bin/sqlplus ods/${PSWD}@${LDAP_CONNECT} @${LDAP_ADMIN}/ldapdncnt.sql >>${INSTALL_LOG} 2>&1

  OIDPROCESS=`head -1 ${TMPDIR}/dncnt.val | sed -e 's* **g'`
  rm -f ${TMPDIR}/dncnt.val
  if [ ${OIDPROCESS} != 0 ]; then
         echo "Directory not empty for bulkload."
         echo "Please empty the directory for bulkload."
         echo 
         exit 1
  fi
fi
# change to LDAP_LOAD directory
cd ${LDAP_LOAD}

if [ -n "${BULKSRC}" -a ! -f "${BULKSRC}" -a \( ${DOCHECK} -ne 0 -o ${DOGENERATE} -ne 0 \) ] ; then
        usage_noreadfile 
        usage
	exit 1
fi

# if check option specified, perform data validation only
if [ ${DOCHECK} != 0 ] ; then 
  echo
  echo -------------------------------------------------------------
  echo Checking data for bulk loading for valid structure...
  echo -------------------------------------------------------------
  echo
  ${LDAP_BIN}/datagen connect=${LDAP_CONNECT} encode=${ENCODING} file=${BULKSRC} schemacheck=TRUE dnfile=${LDAP_LOAD}/dns.dat >${SCHEMACHECK_LOG} 2>&1
  if [ $? != 0 ] ; then
    ERRORS=1
    echo "datagen encountered errors in checking " ${BULKSRC}
    echo "Please correct the errors and restart this script."
    exit 1
  fi
  $ORACLE_HOME/bin/sqlplus ods/${PSWD}@${LDAP_CONNECT} @${LDAP_ADMIN}/ldapbcdt.sql >>${INSTALL_LOG} 2>&1
  (cd ${LDAP_LOAD} ;
  $ORACLE_HOME/bin/sqlldr ods/${PSWD}@${LDAP_CONNECT} control=${LDAP_LOAD}/dns.chk errors=10000 direct=true >>${INSTALL_LOG} 2>&1 )
  (cd ${LDAP_LOG} ; 
  $ORACLE_HOME/bin/sqlplus ods/${PSWD}@${LDAP_CONNECT} @${LDAP_ADMIN}/ldapbcdr.sql >>${INSTALL_LOG} 2>&1 )
  echo 
  echo Schema Check Errors: ${SCHEMACHECK_LOG}
  echo 
  if [ -s ${SCHEMACHECK_LOG} ] ; then
    cat ${SCHEMACHECK_LOG} ;
    ERRORS=1
  else
    echo   None.
  fi
  echo 
  echo Non-Unique Distinguished Names: ${LDAP_LOG}/duplicateDN.log 
  echo 
  if [ -s ${LDAP_LOG}/duplicateDN.log ] ; then
    cat ${LDAP_LOG}/duplicateDN.log
    ERRORS=1
  else
    echo   None.
  fi
  echo
  echo -------------------------------------------------------------
  echo Bulkload data verification complete
  echo -------------------------------------------------------------
  echo
fi

# stop if errors encountered
if [ ${ERRORS} -ne 0 ] ; then
  exit 2;
fi

if [ ${DOGENERATE} != 0 ] ; then 
  cat <<CHECK_SCHEMA_STATE

------------------------------------------------------------------
  Checking Internet Directory current schema state
------------------------------------------------------------------

CHECK_SCHEMA_STATE
  cp $ORACLE_HOME/ldap/admin/ldapbpre.sql $ORACLE_HOME/ldap/admin/ldapbdrp.sql
  chmod 777 $ORACLE_HOME/ldap/admin/ldapbdrp.sql
  if [ $PARALLEL -eq 0 ] ; then
    cp $ORACLE_HOME/ldap/admin/ldapbpst.sql $ORACLE_HOME/ldap/admin/ldapbcre.sql
  else 
  ######################################################################
  # Generating tmp Sql scripts for index creation on dn's etc.- Saheli #  
  # and copy the createCatalogIndexes procedure to ldapbcre.sql  
  ######################################################################
    echo "create index EI_attrStore on DS_AttrStore (entryid)
       tablespace OLTS_IND_ATTRSTORE
       storage (INITIAL 8M NEXT 4M PCTINCREASE 0)
       parallel 2 UNRECOVERABLE;
       commit;
       exit;" >${TMPDIR}/oidindex1.sql

    echo "create unique index EP_dn on CT_dn (entryid, parentdn)
       tablespace OLTS_IND_CT_DN
       storage (INITIAL 4M NEXT 2M PCTINCREASE 0)
       parallel 2 UNRECOVERABLE;
       commit;
       exit;" >${TMPDIR}/oidindex2.sql

    echo "create unique index RP_dn on CT_dn (rdn, parentdn)
       tablespace OLTS_IND_CT_DN
       storage (INITIAL 4M NEXT 2M PCTINCREASE 0)
       parallel 2 UNRECOVERABLE;
       commit;
       exit;" >${TMPDIR}/oidindex3.sql

    echo "create index PN_dn on CT_dn (parentdn)
       tablespace OLTS_IND_CT_DN
       storage (INITIAL 4M NEXT 2M PCTINCREASE 0)
       parallel 2 UNRECOVERABLE;
       commit;
       exit;" >${TMPDIR}/oidindex4.sql

    tail +28 $ORACLE_HOME/ldap/admin/ldapbpst.sql >$ORACLE_HOME/ldap/admin/ldapbcre.sql
    echo "commit;
exit;" >>$ORACLE_HOME/ldap/admin/ldapbcre.sql
  fi

  chmod 777 $ORACLE_HOME/ldap/admin/ldapbcre.sql

  $ORACLE_HOME/bin/sqlplus ods/${PSWD}@${LDAP_CONNECT} @${LDAP_ADMIN}/ldapbnsq.sql >>${INSTALL_LOG} 2>&1

  NEXTSEQ=`cat ${TMPDIR}/nextcatseq.val | sed -e 's* **g'`
  rm -f ${TMPDIR}/nextcatseq.val

  cat <<GENERATE_DATA

------------------------------------------------------------------
  Generating Internet Directory data for bulk loading
------------------------------------------------------------------

GENERATE_DATA
parallel_flag=FALSE
#if [ ${PARALLEL} != 0 ] ; then
#   parallel_flag=TRUE
#fi
if [ ${NODEFLAG} != 0 ] ; then
  ${LDAP_BIN}/datagen connect=${LDAP_CONNECT} encode=${ENCODING} file=${BULKSRC} entrybase=${NEXTSEQ} operational=false creatordn="cn\=bulkload" parallel=${parallel_flag} >>${INSTALL_LOG} 2>&1
else
  ${LDAP_BIN}/datagen connect=${LDAP_CONNECT} encode=${ENCODING} file=${BULKSRC} entrybase=${NEXTSEQ} creatordn="cn\=bulkload" parallel=${parallel_flag} >>${INSTALL_LOG} 2>&1
fi
  if [ $? != 0 ] ; then
        ERRORS=1
	echo "datagen encountered errors in processing " ${BULKSRC}
	echo "Please correct the errors and restart this script."
        exit 1
  fi

  if [ -s ${SCHEMACHECK_LOG} ] ; then
    echo 
    echo Schema Check Errors: ${SCHEMACHECK_LOG}
    echo 
    cat ${SCHEMACHECK_LOG} ;
    DOLOAD=0
  fi
# end if( DOGENERATE != 0 )
fi

# stop here if errors encountered
if [ ${ERRORS} -ne 0 ] ; then
  exit 3;
fi

if [ ${DOLOAD} -ne 0 ] ; then
  cat <<PRELOAD_SCHEMA

cd ${LDAP_LOAD}
------------------------------------------------------------------
 Preparing Internet Directory schema for bulk data loading
------------------------------------------------------------------

PRELOAD_SCHEMA
  $ORACLE_HOME/bin/sqlplus ods/${PSWD}@${LDAP_CONNECT} @${LDAP_ADMIN}/ldapbdrp.sql >>${INSTALL_LOG} 2>&1

  cat <<LOAD_DATA

------------------------------------------------------------------
 Initiating bulk load...
------------------------------------------------------------------

LOAD_DATA

  if [ $PARALLEL != 0 ] ; then
     BCK_GND="&"
     direct_load
     wait
  else
     BCK_GND=""
     direct_load 
  fi
 
  if [ -f ${LDAP_LOG}/*.bad ] ; then
    cat <<DATALOAD_ERRORS

-------------------------------------------------------------
 Done.  Possible Data Loading Errors Encountered.
        Please check ${LDAP_LOG} directory.
-------------------------------------------------------------
DATALOAD_ERRORS
  exit 1;
  fi

  cat <<POSTLOAD_SCHEMA

------------------------------------------------------------------
 Performing post bulk load processing for Internet Directory schema
------------------------------------------------------------------

POSTLOAD_SCHEMA
$ORACLE_HOME/bin/sqlplus ods/${PSWD}@${LDAP_CONNECT} @${LDAP_ADMIN}/ldapbcre.sql >>${INSTALL_LOG} 2>&1
  if [ $PARALLEL != 0 ] ; then
     indexlist=`ls $TMPDIR/oidindex*.sql`
     for indexfile in $indexlist; do
        $ORACLE_HOME/bin/sqlplus ods/${PSWD}@${LDAP_CONNECT} @${indexfile}>>${INSTALL_LOG} 2>&1 & 
     done
     indexlist=`ls $LDAP_LOAD/*.sql`
     for indexfile in $indexlist; do
        $ORACLE_HOME/bin/sqlplus ods/${PSWD}@${LDAP_CONNECT} @${indexfile}>>${INSTALL_LOG} 2>&1 & 
     done

     echo "DROP PROCEDURE createCatalogIndexes;
commit;
exit;" > $TMPDIR/dropProc.sql
     wait
$ORACLE_HOME/bin/sqlplus ods/${PSWD}@${LDAP_CONNECT} @$TMPDIR/dropProc.sql>>${INSTALL_LOG} 2>&1 
     #rm -f $TMPDIR/dropProc.sql
     #rm -f $TMPDIR/oidindex*.sql
  fi 

  # compute adjusted catalog sequence value
  $ORACLE_HOME/bin/sqlplus ods/${PSWD}@${LDAP_CONNECT} @${LDAP_ADMIN}/ldapbmsq.sql >>${INSTALL_LOG} 2>&1
  NEXTSEQVAL=`cat ${TMPDIR}/maxcatseq.val`
  rm -f ${TMPDIR}/maxcatseq.val
  sed -e "s*%NEXTSEQVAL%*${NEXTSEQVAL}*g" ${LDAP_ADMIN}/ldapbnew.orc > ${TMPDIR}/newcatseq.sql
  $ORACLE_HOME/bin/sqlplus ods/${PSWD}@${LDAP_CONNECT} @${TMPDIR}/newcatseq.sql >>${INSTALL_LOG} 2>&1
  rm -f ${TMPDIR}/newcatseq.sql

  if [ -f ${LDAP_ADMIN}/oidstats.sh ]
  then
    sh ${LDAP_ADMIN}/oidstats.sh -connect ${LDAP_CONNECT} -login "ods" -pass ${PSWD}  -cat all -pct 30 > ${TMPDIR}/oidstats.log 2>&1
  fi

 cat <<DATALOAD_DONE

-------------------------------------------------------------
 Done.
-------------------------------------------------------------
DATALOAD_DONE
  # end if( -f LDAP_LOG/*.bad )
    if [ ${NODEFLAG} != 0 ] ; then
      $ORACLE_HOME/bin/sqlplus ods/${PSWD}@${LDAP_CONNECT} @${LDAP_ADMIN}/ldapnncleanup.sql >>${NEWNODE_LOG} 2>&1
    fi

#end if (DOLOAD != 0)
fi

exit 0
