Call Object or Runbatch Crash due to Memory Corruption

CallObject may go to a Zombie or Crashed state when it encounters a bad memory in the business function code. The same behavior is to be expected for a Runbatch process. This crash will produce an extra log file with .dmp.log as its extension. Viewing the .dmp file for a crashed CallObject will show all the threads which were running at the time of the crash. On System-i enterprise server, there will not be any .dmp file. Instead the callstacks for all threads will be logged in jde.log.

The thread which encountered the bad memory and crashed will be highlighted in "red" color and often times will serve as a good starting point for investigation of the bug.

When analyzing the .dmp file, begin by looking for the thread which contains the jdeLogCallStack function, as this is the thread which initiated the crash.

This thread may not always be the root cause of the problem, as it could be a victim of another thread which caused memory corruption but did not crash itself. Crashes of this kind will most likely have a message from the given Operating System in their corresponding jde.logs such as "ACCESS VIOLATION".

This is a dmp.log for a crash due to Memory Corruption:

*****************************************
Tue Mar 30 17:27:28 MDT 2010
*****************************************
Generating call stacks for PID 28774
*****************************************
28774: jdenet_k 6014
----------------- lwp# 1 / thread# 1 --------------------
fc342d74 msgsys (2, 34000023, 13f2d58, 200c, 0, 0)
fc333b84 msgrcv (34000023, 13f2d58, 200c, 0, 0, 0) + 68
ff23e7b4 receiveMessage (5, 34000023, 200c, 0, 0, 0) + 1dc
ff21d73c ipcGetQueueEntry (92d60, ffbfc0c4, ffbfc0c8, ffbfe0d4, 5, 13f2d58) + 334
ff139a24 getExternalQueueEntry (0, 5, 1, ffbff28c, fb3a00ab, ff1d1b58) + 35c
ff195c60 getKernelQueueEntry (0, 8fd68, ffbff28c, 92d60, 8fd54, 8fd58) + 49c
ff19606c processKernelQueue (ffbff28c, fb3a0000, 0, 0, 0, ff1cd2c8) + 300
ff16e250 JDENET_RunKernel (20, 8fd68, 1, ff1cd2c8, 0, 5) + 428
00011cc0 main (0, 0, 4c, 0, 2218c, 0) + 6bc
000111d4 _start (0, 0, 0, 0, 0, 0) + 108
----------------- lwp# 2 / thread# 2 --------------------
fc340408 lwp_park (0, 0, 0)
f6667034 kpuexec (f8e9d800, f1aa34, f14ce0, 0, 0, 18814c) + 2b8
f65c5e50 OCIStmtExecute (188118, f1aa34, 0, 0, 0, 0) + 2c
f657e348 BFOCIStmtExecute (188118, f1aa34, f14ce0, 0, 0, 0) + 28
f656eacc performRequestInternal (116d930, 0, 116d938, 116ca98, ff0ed1c0, 0) + 4f8
f656e46c dballPerformRequest (116d930, 116e920, 0, 116e920, 116e923, 1000) + 3e4
feb937a4 JDB_DBPerformRequest (93bc18, 116d930, 116ca98, 181240, 181318, 181240) + e4
fe13a894 TM_DBPerformRequest (17d050, 1, 116ca98, a6f318, 1b3ab8, 5) + 41c
feba75b4 SelectKeyed (a6f318, 17d050, f635eecc, 1, 1, 1) + 1ec8
febb3c00 FetchKeyed (0, 1, a7cb68, 116ca98, feddd4dc, 0) + 286c
febb114c JDB_FetchKeyed (a6f318, 1, a7cb68, ff338dfc, f636128c, 0) + 214
ee688ab4 EditSystemExistenceF99410 (a7cb68, 689800, f6367910, 3, ff976770, 110c6e0) + 30c
fefb88f8 jdeCallObjectV2 (f1f3d5ec, d38030, a73c40, 3, ff0c36fc, 20) + a3d0
fefae518 jdeCallObject (f1f3d5ec, 0, d38030, 0, 0, 0) + 38
f0b6b804 I4500250_GetGrowerSystemConstant (d38030, a9be40, f6367a80, f6367912, ff990790, 0) + e8
f0b65e50 CalculatePurchasePrice (d38030, a9be40, f6370b3c, f6367a84, ff98bfa8, f6370bda) + 220
fefb88f8 jdeCallObjectV2 (f1ff38fc, d38030, e1af50, 2, ff0c36fc, 20) + a3d0
fefae518 jdeCallObject (f1ff38fc, 0, d38030, 0, 0, 0) + 38
f0f500cc IXTF4311Z1_CalcPurchasePrice (a9be40, f63765ce, 1243948, 1243688, f6376ce8, f63756f0) +
15d8
f0f39fd0 IXT4311Z1_F4311EditLineInternalFunctions (d38030, a9be40, 1243688, f6376f38, 0, f63756f0) +
23b8
fefb88f8 jdeCallObjectV2 (6f7a20, d38030, d28008, 1, ff0c36fc, 20) + a3d0
fefae518 jdeCallObject (6f7a20, 0, d38030, 0, fe963658, f637eea4) + 38
fe79f4dc JDEK_ProcessCallRequest (f637ef58, f076b0, 7d9d08, a80f90, d38030, 20) + 3d30
fe79a5e8 JDEK_StartCallRequest (f637fc40, f076b0, 0, a80f90, 7d9d08, 0) + f74
fe77db24 runBusinessFunction (c44eb8, 0, 0, 7174, ff338dff, fe93c6ec) + 328
fe77dea8 runCallObjectJob (c44eb8, 0, c44ec8, c44ec8, c44ec8, 20) + 5c
fe354394 psthread_pool_worker_function (0, 0, 74, d5cc8, fe10d7b8, fe367de8) + 628
fe0f48fc threadFunctionWrapper (c3548, 204, 0, c4, c354b, fe353d6c) + f4
fc340368 _lwp_start (0, 0, 0, 0, 0, 0)
----------------- lwp# 3 / thread# 3 --------------------
fc340408 lwp_park (0, 0, 0)
fc33a49c cond_wait_queue (2c2350, 2fc040, 0, 0, 0, 0) + 28
fc33aa1c cond_wait (2c2350, 2fc040, 0, 0, 20, 0) + 10
fc33aa58 pthread_cond_wait (2c2350, 2fc040, 0, 0, 20, 1) + 8
fe0f3db8 psthread_cond_wait (2c2148, 2fbe30, 1, 1, 1, fe10d810) + 274
fe352da4 ps_blocking_queue_dequeue (2c1f20, f5ffff24, 2fbe30, 0, fe10d7d8, 20) + 224
fe354214 psthread_pool_worker_function (2c1f20, 0, d5cd4, d5cc8, fe10d7b8, fe367de8) + 4a8
fe0f48fc threadFunctionWrapper (c3588, 204, 0, c4, c358b, fe353d6c) + f4
fc340368 _lwp_start (0, 0, 0, 0, 0, 0)
----------------- lwp# 4 / thread# 4 --------------------
fc340408 lwp_park (0, 0, 0)
f6667034 kpuexec (f8e9d800, f84e4c, f4e190, 0, 0, 18814c) + 2b8
f65c5e50 OCIStmtExecute (188118, f84e4c, 0, 0, 0, 0) + 2c
f657e348 BFOCIStmtExecute (188118, f84e4c, f4e190, 0, 0, 0) + 28
f656eacc performRequestInternal (11c2750, 0, 11c2758, 11c0b10, ff0ed1c0, 0) + 4f8
f656e46c dballPerformRequest (11c2750, 11c3740, 0, 11c3740, 11c3743, 1000) + 3e4
feb937a4 JDB_DBPerformRequest (a6fa80, 11c2750, 11c0b10, 181240, 181318, 181240) + e4
fe13a894 TM_DBPerformRequest (3511d8, 1, 11c0b10, c93e90, 1b3ab8, 5) + 41c
feba75b4 SelectKeyed (c93e90, 3511d8, f58deecc, 1, 1, 1) + 1ec8
febb3c00 FetchKeyed (0, 1, a7ce68, 11c0b10, feddd4dc, 0) + 286c
febb114c JDB_FetchKeyed (c93e90, 1, a7ce68, ff338dfc, f58e128c, 0) + 214
ee688ab4 EditSystemExistenceF99410 (a7ce68, 689800, f58e7910, 3, ff976770, 500e90) + 30c
fefb88f8 jdeCallObjectV2 (f1f3d5ec, 112a060, a73e48, 3, ff0c36fc, 20) + a3d0
fefae518 jdeCallObject (f1f3d5ec, 0, 112a060, 0, 0, 0) + 38
f0b6b804 I4500250_GetGrowerSystemConstant (112a060, 7aa228, f58e7a80, f58e7912, ff990790, 0) + e8
f0b65e50 CalculatePurchasePrice (112a060, 7aa228, f58f0b3c, f58e7a84, ff98bfa8, f58f0bda) + 220
fefb88f8 jdeCallObjectV2 (f1ff38fc, 112a060, f07708, 2, ff0c36fc, 20) + a3d0
fefae518 jdeCallObject (f1ff38fc, 0, 112a060, 0, 0, 0) + 38
f0f500cc IXTF4311Z1_CalcPurchasePrice (7aa228, f58f65ce, a9a898, a9a5d8, f58f6ce8, f58f56f0) + 15d8
f0f39fd0 IXT4311Z1_F4311EditLineInternalFunctions (112a060, 7aa228, a9a5d8, f58f6f38, 0, f58f56f0) + 23b8
fefb88f8 jdeCallObjectV2 (b81968, 112a060, 928860, 1, ff0c36fc, 20) + a3d0
fefae518 jdeCallObject (b81968, 0, 112a060, 0, fe963658, f58feea4) + 38
fe79f4dc JDEK_ProcessCallRequest (f58fef58, 13e3c58, 8a7968, 7471c8, 112a060, 20) + 3d30
fe79a5e8 JDEK_StartCallRequest (f58ffc40, 13e3c58, 0, 7471c8, 8a7968, 0) + f74
fe77db24 runBusinessFunction (1116ff8, 0, 0, 7174, ff338dff, fe93c6ec) + 328
fe77dea8 runCallObjectJob (1116ff8, 0, 1117008, 1117008, 1117008, 20) + 5c
fe354394 psthread_pool_worker_function (0, 0, 74, d5cc8, fe10d7b8, fe367de8) + 628
fe0f48fc threadFunctionWrapper (c35c8, 204, 0, c4, c35cb, fe353d6c) + f4
fc340368 _lwp_start (0, 0, 0, 0, 0, 0)
----------------- lwp# 5 / thread# 5 --------------------
fc341850 waitid (0, 7170, f555c420, 3)
fc334758 waitpid (7170, f555c564, 0, 0, 0, f7fe0c00) + 60
ff30f39c logCallStackOnUnixSigSafe (0, ff316748, f555c581, 24dc, 7170, ff3351f4) + b8
ff310038 jdeLogCallStack (7066, ff338e64, 2400, ff338dfc, 0, ff3351f4) + 1e8  Start Here
ff1470ec krnlSignalHandler (b, 1000, 7066, ff1d217c, 3000, 1154) + 4e0
fc340494 __sighndlr (b, 0, f555ca38, ff146c0c, 0, 1) + c
JD Edwards EnterpriseOne Tools 8.98 Update 3
Kernel Resource Management
Copyright © 2010, Oracle. All rights reserved 27
fc33558c call_user_handler (b, 0, 0, 0, f7fe0c00, f555ca38) + 3b8
f66304ac kpuhhalp (f74bb544, 1774, 16c00, f73eb320, 8048, 16800) + 8ec
f708b174 ttcrbur (182c28, d6784, f662fbc0, 24, f753ac5c, d6898) + 1104
f708b6f8 ttcbur (182c28, d6784, d8b60, 24, 185, 1026) + fc
f6664740 kpuexCallback (182c28, d86e8, f753d1b8, d6784, 1, d8be8) + 1030
f69b36e8 ttcdrv (d86e8, 1894e8, d6784, 182c28, 0, ecb88) + 1da4
f6770b6c nioqwa (0, 189490, f69b1944, d86e8, d67ac, 0) + 40
f65e6f18 upirtrc (d6784, 5e, 0, 182c28, 1, 0) + 544
f66b6380 kpurcsc (188118, 0, 0, d86e8, d93bc, 0) + 6c
f6665c80 kpuexecv8 (188118, 5ad2f8, 5ad344, 17790, 0, f74bb544) + 1390
f66682c0 kpuexec (0, 5ad2f8, f85200, 0, 0, 1000) + 1544
f65c5e50 OCIStmtExecute (188118, 5ad2f8, 0, 0, 0, 0) + 2c
f657e348 BFOCIStmtExecute (188118, 5ad2f8, f85200, 0, 0, 0) + 28
f656eacc performRequestInternal (1257408, 0, 1257410, 11ce838, ff0ed1c0, 0) + 4f8
f656e46c dballPerformRequest (1257408, 12583f8, 0, 12583f8, 12583fb, 1000) + 3e4
feb937a4 JDB_DBPerformRequest (116ec18, 1257408, 11ce838, 181240, 181318, 181240) + e4
fe13a894 TM_DBPerformRequest (9e2d28, 1, 11ce838, f40f18, 4a7e48, 5) + 41c
feba75b4 SelectKeyed (f40f18, 9e2d28, f5564be4, 0, 1, 1) + 1ec8
feba3ed4 JDB_SelectKeyed (f40f18, 7, 0, a000, a118, feddd4dc) + 20c
fe1c969c RTK_CER_FIOSelect (f40f18, 7, 7, f5567620, af7a9c, 8) + ac
f0d5cab4 DetermineIfBlanketPOExists (928a28, c47588, f5571780, 0, f5571814, 24000) + 1934
fefb88f8 jdeCallObjectV2 (f1ff2c98, 928a28, 13fee98, 2, ff0c36fc, 20) + a3d0
fefae518 jdeCallObject (f1ff2c98, 0, 928a28, 0, 0, 0) + 38
f0f420c4 IXTF4311Z1_EditLineMultipleBlanketRelease (928a28, c47588, e5fc28, f5576f38, e5feb6,
f55756f0) + 7c0
f0f39be4 IXT4311Z1_F4311EditLineInternalFunctions (928a28, c47588, e5fc28, 53, 0, f55756f0) + 1fcc
fefb88f8 jdeCallObjectV2 (b81788, 928a28, 1106270, 1, ff0c36fc, 20) + a3d0
fefae518 jdeCallObject (b81788, 0, 928a28, 0, fe963658, f557eea4) + 38
fe79f4dc JDEK_ProcessCallRequest (f557ef58, dda5b8, 5a7e68, 9e2ae8, 928a28, 20) + 3d30
fe79a5e8 JDEK_StartCallRequest (f557fc40, dda5b8, 0, 9e2ae8, 5a7e68, 0) + f74
fe77db24 runBusinessFunction (c59d88, 0, 0, 7174, ff338dff, fe93c6ec) + 328
fe77dea8 runCallObjectJob (c59d88, 0, c59d98, c59d98, c59d98, 20) + 5c
fe354394 psthread_pool_worker_function (0, 0, 74, d5cc8, fe10d7b8, fe367de8) + 628
fe0f48fc threadFunctionWrapper (c3478, 204, 0, c4, c347b, fe353d6c) + f4
fc340368 _lwp_start (0, 0, 0, 0, 0, 0)
----------------- lwp# 6 / thread# 6 --------------------
fc340408 lwp_park (0, 0, 0)
f6667034 kpuexec (f8e9d800, f1492c, 5ad6ac, 0, 0, 18814c) + 2b8
f65c5e50 OCIStmtExecute (188118, f1492c, 0, 0, 0, 0) + 2c
f657e348 BFOCIStmtExecute (188118, f1492c, 5ad6ac, 0, 0, 0) + 28
f656eacc performRequestInternal (d05080, 0, d05088, d03440, ff0ed1c0, 0) + 4f8
f656e46c dballPerformRequest (d05080, d06070, 0, d06070, d06073, 1000) + 3e4
feb937a4 JDB_DBPerformRequest (93bc50, d05080, d03440, 181240, 181318, 181240) + e4
fe13a894 TM_DBPerformRequest (9e2a20, 1, d03440, 116e990, 1b3ab8, 5) + 41c
feba75b4 SelectKeyed (116e990, 9e2a20, f385eecc, 1, 1, 1) + 1ec8
febb3c00 FetchKeyed (0, 1, 1562d8, d03440, feddd4dc, 0) + 286c
febb114c JDB_FetchKeyed (116e990, 1, 1562d8, ff338dfc, f386128c, 0) + 214
ee688ab4 EditSystemExistenceF99410 (1562d8, 689800, f3867910, 3, ff976770, 618700) + 30c
fefb88f8 jdeCallObjectV2 (f1f3d5ec, 13741c8, a6fb20, 3, ff0c36fc, 20) + a3d0
fefae518 jdeCallObject (f1f3d5ec, 0, 13741c8, 0, 0, 0) + 38
f0b6b804 I4500250_GetGrowerSystemConstant (13741c8, b86dd8, f3867a80, f3867912, ff990790, 0) + e8
f0b65e50 CalculatePurchasePrice (13741c8, b86dd8, f3870b3c, f3867a84, ff98bfa8, f3870bda) + 220
fefb88f8 jdeCallObjectV2 (f1ff38fc, 13741c8, 117a878, 2, ff0c36fc, 20) + a3d0
fefae518 jdeCallObject (f1ff38fc, 0, 13741c8, 0, 0, 0) + 38
f0f500cc IXTF4311Z1_CalcPurchasePrice (b86dd8, f38765ce, 1265f00, 1265c40, f3876ce8, f38756f0) + 15d8
f0f39fd0 IXT4311Z1_F4311EditLineInternalFunctions (13741c8, b86dd8, 1265c40, f3876f38, 0, f38756f0) + 23b8
fefb88f8 jdeCallObjectV2 (6f7908, 13741c8, eb79e8, 1, ff0c36fc, 20) + a3d0
fefae518 jdeCallObject (6f7908, 0, 13741c8, 0, fe963658, f387eea4) + 38
fe79f4dc JDEK_ProcessCallRequest (f387ef58, 13e3c78, bdb60, 9e2768, 13741c8, 20) + 3d30
fe79a5e8 JDEK_StartCallRequest (f387fc40, 13e3c78, 0, 9e2768, bdb60, 0) + f74
fe77db24 runBusinessFunction (14c288, 0, 0, 7174, ff338dff, fe93c6ec) + 328
fe77dea8 runCallObjectJob (14c288, 0, 14c298, 14c298, 14c298, 20) + 5c
fe354394 psthread_pool_worker_function (0, 0, 74, d5cc8, fe10d7b8, fe367de8) + 628
fe0f48fc threadFunctionWrapper (36cdb8, 204, 0, c4, 36cdbb, fe353d6c) + f4
fc340368 _lwp_start (0, 0, 0, 0, 0, 0)