Printable Version of Topic Click here to view this topic in its original format

Unofficial VirtualDub Support Forums > Testing / Bug Reports > An Instruction Not Supported By The Cpu Was Exe

Posted by: Galvin Sep 15 2012, 06:45 PM

Windows 7 64bit SP1 Using ivy bridge 3770K cpu Virtual dub 1.9.11 (coundn't find a newer release on the download page) X264 coded x264vfw64_37_2200bm_33968 CODE VirtualDub crash report -- build 32842 (release-AMD64) -------------------------------------- Disassembly: 7fefbf43cc0: f5              cmc     7fefbf43cc1: 0f62ed          punpckldq mm5, mm5 7fefbf43cc4: 0f6af6          punpckhdq mm6, mm6 7fefbf43cc7: 0fede8          paddsw mm5, mm0 7fefbf43cca: 0fedf1          paddsw mm6, mm1 7fefbf43ccd: 0f67ee          packuswb mm5, mm6 7fefbf43cd0: 0fdbec          pand   mm5, mm4 7fefbf43cd3: 0f7f6f10        movq   [rdi+10h], mm5 7fefbf43cd7: 0f69ff          punpckhwd mm7, mm7 7fefbf43cda: 0f7ffd          movq   mm5, mm7 7fefbf43cdd: 0f62ed          punpckldq mm5, mm5 7fefbf43ce0: 0f6aff          punpckhdq mm7, mm7 7fefbf43ce3: 0fedea          paddsw mm5, mm2 7fefbf43ce6: 0fedfb          paddsw mm7, mm3 7fefbf43ce9: 0f67ef          packuswb mm5, mm7 7fefbf43cec: 0fdbec          pand   mm5, mm4 7fefbf43cef: 0f7f6f18        movq   [rdi+18h], mm5 7fefbf43cf3: 4883c720        add    rdi, 20h 7fefbf43cf7: 4883ee08        sub    rsi, 08h 7fefbf43cfb: 0f8531feffff    jnz    fbf43b32 7fefbf43d01: 492bfa          sub    edi, edx 7fefbf43d04: 4983e901        sub    rcx, 01h 7fefbf43d08: 0f8521feffff    jnz    fbf43b2f 7fefbf43d0e: 0f77            emms   7fefbf43d10: 415a            pop    edx 7fefbf43d12: 5b              pop    ebx 7fefbf43d13: 5f              pop    edi 7fefbf43d14: 5e              pop    esi 7fefbf43d15: c3              ret     7fefbf43d16: 4863442428      movsxd rax, [rsp+28h] 7fefbf43d1b: 56              push   esi 7fefbf43d1c: 57              push   edi 7fefbf43d1d: 53              push   ebx 7fefbf43d1e: 4152            push   edx 7fefbf43d20: 4d63c0          movsxd r8, eax 7fefbf43d23: 4d63c9          movsxd r9, ecx 7fefbf43d26: 488bd8          mov    rbx, eax 7fefbf43d29: 4f8d1480        lea    r10, [r8+r8*4] 7fefbf43d2d: 4d03d0          add    edx, eax 7fefbf43d30: 488bc1          mov    rax, ecx 7fefbf43d33: 488bfa          mov    rdi, edx 7fefbf43d36: 498bd0          mov    rdx, eax 7fefbf43d39: 488bf2          mov    rsi, edx 7fefbf43d3c: 0f6e3c03        movd   mm7, [rbx+rax] 7fefbf43d40: 0fefc0          pxor   mm0, mm0 7fefbf43d43: 0f6e3458        movd   mm6, [rax+rbx*2] 7fefbf43d47: 0f6f0d72e50000  movq   mm1, [fbf522c0] 7fefbf43d4e: 0f6f1573e50000  movq   mm2, [fbf522c8] 7fefbf43d55: 0f60fe          punpcklbw mm7, mm6 7fefbf43d58: 0f7ffe          movq   mm6, mm7 7fefbf43d5b: 0f60f0          punpcklbw mm6, mm0 7fefbf43d5e: 0ff9f2          psubw  mm6, mm2 7fefbf43d61: 0f68f8          punpckhbw mm7, mm0 7fefbf43d64: 0ff9fa          psubw  mm7, mm2 7fefbf43d67: 0f7ff2          movq   mm2, mm6 7fefbf43d6a: 0f61d0          punpcklwd mm2, mm0 7fefbf43d6d: 0ff5d1          pmaddwd mm2, mm1 7fefbf43d70: 0f7ff3          movq   mm3, mm6 7fefbf43d73: 0f69d8          punpckhwd mm3, mm0 7fefbf43d76: 0ff5d9          pmaddwd mm3, mm1 7fefbf43d79: 0f7ffc          movq   mm4, mm7 7fefbf43d7c: 0f61e0          punpcklwd mm4, mm0 7fefbf43d7f: 0ff5e1          pmaddwd mm4, mm1 7fefbf43d82: 0f7ffd          movq   mm5, mm7 7fefbf43d85: 0f69e8          punpckhwd mm5, mm0 7fefbf43d88: 0f6f0529e50000  movq   mm0, [fbf522b8] 7fefbf43d8f: 0ff5e9          pmaddwd mm5, mm1 7fefbf43d92: 0f72e20e        psrad  mm2, 0eh 7fefbf43d96: 0ff5f0          pmaddwd mm6, mm0 7fefbf43d99: 0f72e30e        psrad  mm3, 0eh 7fefbf43d9d: 0ff5f8          pmaddwd mm7, mm0 7fefbf43da0: 0f72e40e        psrad  mm4, 0eh 7fefbf43da4: 0f6bd3          packssdw mm2, mm3 7fefbf43da7: 0f72e50e        psrad  mm5, 0eh 7fefbf43dab: 0f6be5          packssdw mm4, mm5 7fefbf43dae: 0f72e60e        psrad  mm6, 0eh 7fefbf43db2: 0f7fd0          movq   mm0, mm2 7fefbf43db5: 0f61c6          punpcklwd mm0, mm6 7fefbf43db8: 0f7fd1          movq   mm1, mm2 7fefbf43dbb: 0f69ce          punpckhwd mm1, mm6 7fefbf43dbe: 0f              db     0fh 7fefbf43dbf: 72              db     72h Built on Aegis on Fri Dec 24 13:19:54 2010 using compiler version 1400 Windows 6.1 (Windows Vista x64 build 7601) [Service Pack 1] RAX =         1fe9e0e8 RBX =           232880 RCX =             2922 RDX =              780 RSI =              398 RDI =          63dd6b8 RBP =                0 R8  =              780 R9  =              20a R10 =             2d00 R11 =           232800 R12 =                0 R13 =                0 R14 =                0 R15 =                0 RSP =          564fee8 RIP =      7fefbf43d56 EFLAGS = 00010206 Crash reason: Illegal instruction Crash context: An instruction not supported by the CPU was executed in module 'Veedub64'. Pointer dumps: RAX   1fe9e0e8: 1a151f22 1616191f 16161616 17161616 1a1c1c1e 17161b1b 17171617 18171717 RBX   00232880: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000 RDI   063dd6b8: 00161622 16001616 17170016 00171700 17001717 19170017 02191700 1f03241e RSP   0564fee8: 0644a600 00000000 1fd60000 00000000 00000780 00000000 03dc0000 00000000      0564ff08: fbf429ae 000007fe 00000780 00000000 00000000 00000000 1fd60000 00000000      0564ff28: 00000000 00000000 00232880 00000000 00000000 00000000 00000458 00000000      0564ff48: 0000025c 00000000 00000000 00000000 776d652d 00000000 00000000 00000000 R11   00232800: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000 Thread call stack: 7fefbf43d56: frapsv64!DriverProc [7fefbf40000+1e20+1f36] 7fefbf429ae: frapsv64!DriverProc [7fefbf40000+1e20+b8e] 776d652d: kernel32!BaseThreadInitThunk [776c0000+16520+d] 77c7c521: ntdll!RtlUserThreadStart [77c50000+2c500+21] -- End of report

Posted by: dloneranger Sep 15 2012, 07:20 PM

What version of fraps do you have - it looks like that's the bit that's actually crashing The current version is Fraps 3.5.9 and it's fixed a few bugs that can cause crashes in x64 and editing

Posted by: Galvin Sep 16 2012, 05:51 AM

3.5.7 I'll update to 3.5.9 Thanks

Posted by: Galvin Sep 16 2012, 08:21 AM

Updated to the latest fraps, still getting the error. Maybe its file size this one is around 17GB CODE VirtualDub crash report -- build 32842 (release-AMD64) -------------------------------------- Disassembly: 7fefbe83cc0: f5              cmc     7fefbe83cc1: 0f62ed          punpckldq mm5, mm5 7fefbe83cc4: 0f6af6          punpckhdq mm6, mm6 7fefbe83cc7: 0fede8          paddsw mm5, mm0 7fefbe83cca: 0fedf1          paddsw mm6, mm1 7fefbe83ccd: 0f67ee          packuswb mm5, mm6 7fefbe83cd0: 0fdbec          pand   mm5, mm4 7fefbe83cd3: 0f7f6f10        movq   [rdi+10h], mm5 7fefbe83cd7: 0f69ff          punpckhwd mm7, mm7 7fefbe83cda: 0f7ffd          movq   mm5, mm7 7fefbe83cdd: 0f62ed          punpckldq mm5, mm5 7fefbe83ce0: 0f6aff          punpckhdq mm7, mm7 7fefbe83ce3: 0fedea          paddsw mm5, mm2 7fefbe83ce6: 0fedfb          paddsw mm7, mm3 7fefbe83ce9: 0f67ef          packuswb mm5, mm7 7fefbe83cec: 0fdbec          pand   mm5, mm4 7fefbe83cef: 0f7f6f18        movq   [rdi+18h], mm5 7fefbe83cf3: 4883c720        add    rdi, 20h 7fefbe83cf7: 4883ee08        sub    rsi, 08h 7fefbe83cfb: 0f8531feffff    jnz    fbe83b32 7fefbe83d01: 492bfa          sub    edi, edx 7fefbe83d04: 4983e901        sub    rcx, 01h 7fefbe83d08: 0f8521feffff    jnz    fbe83b2f 7fefbe83d0e: 0f77            emms   7fefbe83d10: 415a            pop    edx 7fefbe83d12: 5b              pop    ebx 7fefbe83d13: 5f              pop    edi 7fefbe83d14: 5e              pop    esi 7fefbe83d15: c3              ret     7fefbe83d16: 4863442428      movsxd rax, [rsp+28h] 7fefbe83d1b: 56              push   esi 7fefbe83d1c: 57              push   edi 7fefbe83d1d: 53              push   ebx 7fefbe83d1e: 4152            push   edx 7fefbe83d20: 4d63c0          movsxd r8, eax 7fefbe83d23: 4d63c9          movsxd r9, ecx 7fefbe83d26: 488bd8          mov    rbx, eax 7fefbe83d29: 4f8d1480        lea    r10, [r8+r8*4] 7fefbe83d2d: 4d03d0          add    edx, eax 7fefbe83d30: 488bc1          mov    rax, ecx 7fefbe83d33: 488bfa          mov    rdi, edx 7fefbe83d36: 498bd0          mov    rdx, eax 7fefbe83d39: 488bf2          mov    rsi, edx 7fefbe83d3c: 0f6e3c03        movd   mm7, [rbx+rax] 7fefbe83d40: 0fefc0          pxor   mm0, mm0 7fefbe83d43: 0f6e3458        movd   mm6, [rax+rbx*2] 7fefbe83d47: 0f6f0d72e50000  movq   mm1, [fbe922c0] 7fefbe83d4e: 0f6f1573e50000  movq   mm2, [fbe922c8] 7fefbe83d55: 0f60fe          punpcklbw mm7, mm6 7fefbe83d58: 0f7ffe          movq   mm6, mm7 7fefbe83d5b: 0f60f0          punpcklbw mm6, mm0 7fefbe83d5e: 0ff9f2          psubw  mm6, mm2 7fefbe83d61: 0f68f8          punpckhbw mm7, mm0 7fefbe83d64: 0ff9fa          psubw  mm7, mm2 7fefbe83d67: 0f7ff2          movq   mm2, mm6 7fefbe83d6a: 0f61d0          punpcklwd mm2, mm0 7fefbe83d6d: 0ff5d1          pmaddwd mm2, mm1 7fefbe83d70: 0f7ff3          movq   mm3, mm6 7fefbe83d73: 0f69d8          punpckhwd mm3, mm0 7fefbe83d76: 0ff5d9          pmaddwd mm3, mm1 7fefbe83d79: 0f7ffc          movq   mm4, mm7 7fefbe83d7c: 0f61e0          punpcklwd mm4, mm0 7fefbe83d7f: 0ff5e1          pmaddwd mm4, mm1 7fefbe83d82: 0f7ffd          movq   mm5, mm7 7fefbe83d85: 0f69e8          punpckhwd mm5, mm0 7fefbe83d88: 0f6f0529e50000  movq   mm0, [fbe922b8] 7fefbe83d8f: 0ff5e9          pmaddwd mm5, mm1 7fefbe83d92: 0f72e20e        psrad  mm2, 0eh 7fefbe83d96: 0ff5f0          pmaddwd mm6, mm0 7fefbe83d99: 0f72e30e        psrad  mm3, 0eh 7fefbe83d9d: 0ff5f8          pmaddwd mm7, mm0 7fefbe83da0: 0f72e40e        psrad  mm4, 0eh 7fefbe83da4: 0f6bd3          packssdw mm2, mm3 7fefbe83da7: 0f72e50e        psrad  mm5, 0eh 7fefbe83dab: 0f6be5          packssdw mm4, mm5 7fefbe83dae: 0f72e60e        psrad  mm6, 0eh 7fefbe83db2: 0f7fd0          movq   mm0, mm2 7fefbe83db5: 0f61c6          punpcklwd mm0, mm6 7fefbe83db8: 0f7fd1          movq   mm1, mm2 7fefbe83dbb: 0f69ce          punpckhwd mm1, mm6 7fefbe83dbe: 0f              db     0fh 7fefbe83dbf: 72              db     72h Built on Aegis on Fri Dec 24 13:19:54 2010 using compiler version 1400 Windows 6.1 (Windows Vista x64 build 7601) [Service Pack 1] RAX =         20fd41e0 RBX =           232880 RCX =             2d2b RDX =              780 RSI =              2a0 RDI =          65db9a0 RBP =              258 R8  =              780 R9  =               72 R10 =             2d00 R11 =           232880 R12 =         37da0008 R13 =                0 R14 =                0 R15 =                0 RSP =          61dfee8 RIP =      7fefbe83d56 EFLAGS = 00010202 Crash reason: Illegal instruction Crash context: An instruction not supported by the CPU was executed in module 'Veedub64'. Pointer dumps: RAX   20fd41e0: 29292828 3b28282a 24252732 2b2a2322 28282b2c 302e2b29 2d2f3031 2b2a2a2b RBX   00232880: 00000000 00000000 00000000 00000000 00000000 00000000 27a41330 00000000 RDI   065db9a0: 012c2a2c 2b022c2a 2d2b022d 012d2b02 29012f29 473b1431 0040310b 24003626 RSP   061dfee8: 001a5860 00000000 20ef0000 00000000 00000780 00000000 01f10000 00000000      061dff08: fbe82c9c 000007fe 37da0008 00000000 00000000 00000000 000003c0 00000000      061dff28: 00000258 00000000 00232880 00000000 00000000 00000000 0000053c 00000000      061dff48: 000003f8 00000000 00000000 00000000 776d652d 00000000 00000000 00000000 R11   00232880: 00000000 00000000 00000000 00000000 00000000 00000000 27a41330 00000000 R12   37da0008: 78535046 00000010 000d86f8 00101cf0 000f1eb5 000549a0 0001ff31 0000cb68 Thread call stack: 7fefbe83d56: frapsv64!DriverProc [7fefbe80000+1e20+1f36] 7fefbe82c9c: frapsv64!DriverProc [7fefbe80000+1e20+e7c] 776d652d: kernel32!BaseThreadInitThunk [776c0000+16520+d] 77c7c521: ntdll!RtlUserThreadStart [77c50000+2c500+21] -- End of report

Posted by: phaeron Sep 23 2012, 08:29 PM

Same crash as before. Hard to tell what's going on, but it's definitely still a problem in the fraps codec. Looks like they might have a jump table gone wrong, but in that case it'd be a mystery how it worked at all -- an Ivy Bridge CPU is pretty new and there aren't many x86 CPUs that can execute instructions it can't.

Posted by: dloneranger Sep 23 2012, 09:17 PM

QUOTE Maybe its file size this one is around 17GB Nah, that's tiny ;-) What happens if you use the 32 bit version of virtualdub and frap's 32 bit codec?

Powered by Invision Board (http://www.invisionboard.com)
© Invision Power Services (http://www.invisionpower.com)