The problem is something a bit else (jstarks figured it out somewhere below). I'm not a compiler/abi eng, but it seems to depend on a compiler, eg. consider this with clang-16:
#include <sys/syscall.h>
#include <unistd.h>
#include <alloca.h>
#include <string.h>
void s(long a, long b, long c, long d, long e, long f, long g) {
}
int main(void) {
long a = 0xFFFFFFFFFFFFFFFF;
s(a, a, a, a, a, a, a);
syscall(9999, 1, 2, 3, 4, 5, 6);
return 0;
}
Now, strace shows: $ strace -e process_vm_readv ./a
process_vm_readv(1, 0x2, 3, 0x4, 5, 18446744069414584326) = -1 EINVAL (Invalid argument)
objdump -d a 117f: 48 c7 45 f0 ff ff ff movq $0xffffffffffffffff,-0x10(%rbp)
1186: ff
1187: 48 8b 7d f0 mov -0x10(%rbp),%rdi
118b: 48 8b 75 f0 mov -0x10(%rbp),%rsi
118f: 48 8b 55 f0 mov -0x10(%rbp),%rdx
1193: 48 8b 4d f0 mov -0x10(%rbp),%rcx
1197: 4c 8b 45 f0 mov -0x10(%rbp),%r8
119b: 4c 8b 4d f0 mov -0x10(%rbp),%r9
119f: 48 8b 45 f0 mov -0x10(%rbp),%rax
11a3: 48 89 04 24 mov %rax,(%rsp)
11a7: e8 94 ff ff ff call 1140 <s>
11ac: bf 36 01 00 00 mov $0x136,%edi
11b1: be 01 00 00 00 mov $0x1,%esi
11b6: ba 02 00 00 00 mov $0x2,%edx
11bb: b9 03 00 00 00 mov $0x3,%ecx
11c0: 41 b8 04 00 00 00 mov $0x4,%r8d
11c6: 41 b9 05 00 00 00 mov $0x5,%r9d
11cc: c7 04 24 06 00 00 00 movl $0x6,(%rsp)
11d3: b0 00 mov $0x0,%al
11d5: e8 56 fe ff ff call 1030 <syscall@plt>
Only 4 bytes are put on the stack, but syscall will read 8.It's tricky if one doesn't control types of arguments used in vararg.