[Date Prev][Date Next] [Chronological] [Thread] [Top]

data import very slow for data migration



Hi

We are having 4 million users to migrate, all data exported from oracle to multiple ldif files.
Imported 1 million till now, took almost 28 hours. and openldap-data dir of about 28G.
openldap version 2.4.33 bdb version 5.1.29  RHEL 6.3 RAM 8G 4 cpu , system is a VM.

Currently running slapadd output
+ /apps/openldap/sbin/slapadd -q -c -w -f /apps/openldap/etc/openldap/slapd.conf -l /root/User9.ldif
bdb_monitor_db_open: monitoring disabled; configure monitor database to enable
.                       2.27% eta 21h31m elapsed          29m57s spd   1.6 k/s str2entry: invalid value for attributeType postalAddress #0 (syntax 1.3.6.1.4.1.1466.115.121.1.41)
slapadd: could not parse entry (line=394416)
*                       2.81% eta 19h59m elapsed          34m40s spd  10.1 k/s

Its seems to be taking weeks go import whole data.

is there  any tool or any other approach which we can use to make it fast,Or we are going with wrong configuration.
Or we have to switch to ODS or RHDS

Top output
top - 10:26:04 up 21 days,  6:51,  3 users,  load average: 2.13, 2.06, 1.79
Tasks: 153 total,   2 running, 151 sleeping,   0 stopped,   0 zombie
Cpu0  :100.0%us,  0.0%sy,  0.0%ni,  0.0%id,  0.0%wa,  0.0%hi,  0.0%si,  0.0%st
Cpu1  :  0.0%us,  0.0%sy,  0.0%ni,100.0%id,  0.0%wa,  0.0%hi,  0.0%si,  0.0%st
Cpu2  :  0.0%us,  0.0%sy,  0.0%ni,100.0%id,  0.0%wa,  0.0%hi,  0.0%si,  0.0%st
Cpu3  :  3.0%us,  0.3%sy,  0.0%ni,  0.0%id, 96.6%wa,  0.0%hi,  0.0%si,  0.0%st
Mem:   9095980k total,  8956852k used,   139128k free,    31452k buffers
Swap:  6291448k total,    21300k used,  6270148k free,  7431012k cached

  PID USER      PR  NI  VIRT  RES  SHR S %CPU %MEM    TIME+  COMMAND                                                           
27877 ldap      20   0 4855m 539m 284m S 99.8  6.1   1807:28 slapd                                                              
21130 root      20   0 5267m 3.8g 3.0g R  4.3 43.6   0:59.27 slapadd 

DB_CONFIG

set_cachesize   0       4294967295      0
set_lg_regionmax        2048576
set_lg_max              20485760
set_lg_bsize            2097152
set_lk_max_locks 10000
set_lk_max_objects 5000
set_lk_max_lockers 5000

slapd.conf

include        /apps/openldap/etc/openldap/schema/core.schema
include        /apps/openldap/etc/openldap/schema/cosine.schema
include        /apps/openldap/etc/openldap/schema/nis.schema
include        /apps/openldap/etc/openldap/schema/inetorgperson.schema
include        /apps/openldap/etc/openldap/schema/openldap.schema
include        /apps/openldap/etc/openldap/schema/dyngroup.schema
include        /apps/openldap/etc/openldap/schema/ppolicy.schema
include        /apps/openldap/etc/openldap/schema/channelIdentifier.schema
include        /apps/openldap/etc/openldap/schema/platform.schema
include        /apps/openldap/etc/openldap/schema/extendedProfileKey.schema
include        /apps/openldap/etc/openldap/schema/extendedProfileValue.schema
include        /apps/openldap/etc/openldap/schema/behaviorKey.schema
include        /apps/openldap/etc/openldap/schema/behaviorValue.schema
include        /apps/openldap/etc/openldap/schema/questionAnswer.schema
include        /apps/openldap/etc/openldap/schema/extendedTop.schema
include        /apps/openldap/etc/openldap/schema/counter.schema


pidfile        /apps/openldap/var/run/slapd.pid
argsfile    /apps/openldap/var/run/slapd.args


logfile /apps/logs/ldap
loglevel 16640


database    bdb
suffix        "dc=example,dc=com"

access to attrs=userPassword
      by self     write
      by anonymous auth
      by * break          
                 
access to *
    by group/groupOfUniqueNames/uniqueMember.exact="cn=VWrite,ou=businessUsersGroup,dc=example,dc=com" manage
    by group/groupOfUniqueNames/uniqueMember.exact="cn=VRead,ou=businessUsersGroup,dc=example,dc=com" read
    by * break

access to *
      by self       write
      by anonymous  auth
      by *          read    


rootdn        "cn=Manager,dc=example,dc=com"
rootpw  {SSHA}dXDFSQeFjSofJ3TAzYf8DrDSYWY

##################  SSL  ##########################################
#
TLSCipherSuite HIGH:MEDIUM:+SSLv3
TLSCACertificateFile /apps/openldap/etc/openldap/cacerts/cacert.pem
TLSCertificateFile /apps/openldap/etc/openldap/cacerts/dam01.crt
TLSCertificateKeyFile /apps/openldap/etc/openldap/cacerts/dam01.key
#
####################################################################

####ache Entries #####
cachesize 900000
#idlcachesize 900000
lastmod         on
checkpoint 128 15
concurrency 100
index entryCSN eq
index entryUUID eq
index mail,uid,postalCode,smail,channelType,channelValue,answer,behavName,objectclass,tokenID,type eq
index givenName,sn,city,question,behavValue,cn,extName sub
index displayName approx
# Replication Configuration
overlay syncprov
syncprov-checkpoint 100 10
syncprov-sessionlog 100

serverid        1

syncrepl        rid=111
                provider=ldap://s01.com
                binddn="cn=Manager,dc=example,dc=com"
                bindmethod=simple
                starttls=yes
                tls_reqcert=allow
        schemachecking=off
                credentials=G00gle#
                searchbase="dc=example,dc=com"
                type=refreshAndPersist
                retry="5 5 300 +"
                interval=00:00:00:10

syncrepl        rid=222
                provider=ldap://m04.com
                binddn="cn=Manager,dc=example,dc=com"
                bindmethod=simple
                starttls=yes
                tls_reqcert=allow
        schemachecking=off
                credentials=G00gle#
                searchbase="dc=example,dc=com"
                type=refreshAndPersist
                retry="5 5 300 +"
                interval=00:00:00:10

######

mirrormode TRUE

directory    /apps/openldap/var/openldap-data

overlay unique
unique_attributes mail

overlay ppolicy
ppolicy_default "cn=default,ou=pwdPolicy,dc=example,dc=com"
ppolicy_use_lockout
--

Please let me know in case you need further details.


Thanks&Regards
Anil Beniwal
+919891695048