1/*
2** FFI C callback handling.
3** Copyright (C) 2005-2021 Mike Pall. See Copyright Notice in luajit.h
4*/
5
6#include "lj_obj.h"
7
8#if LJ_HASFFI
9
10#include "lj_gc.h"
11#include "lj_err.h"
12#include "lj_tab.h"
13#include "lj_state.h"
14#include "lj_frame.h"
15#include "lj_ctype.h"
16#include "lj_cconv.h"
17#include "lj_ccall.h"
18#include "lj_ccallback.h"
19#include "lj_target.h"
20#include "lj_mcode.h"
21#include "lj_trace.h"
22#include "lj_vm.h"
23
24/* -- Target-specific handling of callback slots -------------------------- */
25
26#define CALLBACK_MCODE_SIZE (LJ_PAGESIZE * LJ_NUM_CBPAGE)
27
28#if LJ_OS_NOJIT
29
30/* Callbacks disabled. */
31#define CALLBACK_SLOT2OFS(slot) (0*(slot))
32#define CALLBACK_OFS2SLOT(ofs) (0*(ofs))
33#define CALLBACK_MAX_SLOT 0
34
35#elif LJ_TARGET_X86ORX64
36
37#define CALLBACK_MCODE_HEAD (LJ_64 ? 8 : 0)
38#define CALLBACK_MCODE_GROUP (-2+1+2+(LJ_GC64 ? 10 : 5)+(LJ_64 ? 6 : 5))
39
40#define CALLBACK_SLOT2OFS(slot) \
41 (CALLBACK_MCODE_HEAD + CALLBACK_MCODE_GROUP*((slot)/32) + 4*(slot))
42
43static MSize CALLBACK_OFS2SLOT(MSize ofs)
44{
45 MSize group;
46 ofs -= CALLBACK_MCODE_HEAD;
47 group = ofs / (32*4 + CALLBACK_MCODE_GROUP);
48 return (ofs % (32*4 + CALLBACK_MCODE_GROUP))/4 + group*32;
49}
50
51#define CALLBACK_MAX_SLOT \
52 (((CALLBACK_MCODE_SIZE-CALLBACK_MCODE_HEAD)/(CALLBACK_MCODE_GROUP+4*32))*32)
53
54#elif LJ_TARGET_ARM
55
56#define CALLBACK_MCODE_HEAD 32
57
58#elif LJ_TARGET_ARM64
59
60#define CALLBACK_MCODE_HEAD 32
61
62#elif LJ_TARGET_PPC
63
64#define CALLBACK_MCODE_HEAD 24
65
66#elif LJ_TARGET_MIPS32
67
68#define CALLBACK_MCODE_HEAD 20
69
70#elif LJ_TARGET_MIPS64
71
72#define CALLBACK_MCODE_HEAD 52
73
74#else
75
76/* Missing support for this architecture. */
77#define CALLBACK_SLOT2OFS(slot) (0*(slot))
78#define CALLBACK_OFS2SLOT(ofs) (0*(ofs))
79#define CALLBACK_MAX_SLOT 0
80
81#endif
82
83#ifndef CALLBACK_SLOT2OFS
84#define CALLBACK_SLOT2OFS(slot) (CALLBACK_MCODE_HEAD + 8*(slot))
85#define CALLBACK_OFS2SLOT(ofs) (((ofs)-CALLBACK_MCODE_HEAD)/8)
86#define CALLBACK_MAX_SLOT (CALLBACK_OFS2SLOT(CALLBACK_MCODE_SIZE))
87#endif
88
89/* Convert callback slot number to callback function pointer. */
90static void *callback_slot2ptr(CTState *cts, MSize slot)
91{
92 return (uint8_t *)cts->cb.mcode + CALLBACK_SLOT2OFS(slot);
93}
94
95/* Convert callback function pointer to slot number. */
96MSize lj_ccallback_ptr2slot(CTState *cts, void *p)
97{
98 uintptr_t ofs = (uintptr_t)((uint8_t *)p -(uint8_t *)cts->cb.mcode);
99 if (ofs < CALLBACK_MCODE_SIZE) {
100 MSize slot = CALLBACK_OFS2SLOT((MSize)ofs);
101 if (CALLBACK_SLOT2OFS(slot) == (MSize)ofs)
102 return slot;
103 }
104 return ~0u; /* Not a known callback function pointer. */
105}
106
107/* Initialize machine code for callback function pointers. */
108#if LJ_OS_NOJIT
109/* Disabled callback support. */
110#define callback_mcode_init(g, p) (p)
111#elif LJ_TARGET_X86ORX64
112static void *callback_mcode_init(global_State *g, uint8_t *page)
113{
114 uint8_t *p = page;
115 uint8_t *target = (uint8_t *)(void *)lj_vm_ffi_callback;
116 MSize slot;
117#if LJ_64
118 *(void **)p = target; p += 8;
119#endif
120 for (slot = 0; slot < CALLBACK_MAX_SLOT; slot++) {
121 /* mov al, slot; jmp group */
122 *p++ = XI_MOVrib | RID_EAX; *p++ = (uint8_t)slot;
123 if ((slot & 31) == 31 || slot == CALLBACK_MAX_SLOT-1) {
124 /* push ebp/rbp; mov ah, slot>>8; mov ebp, &g. */
125 *p++ = XI_PUSH + RID_EBP;
126 *p++ = XI_MOVrib | (RID_EAX+4); *p++ = (uint8_t)(slot >> 8);
127#if LJ_GC64
128 *p++ = 0x48; *p++ = XI_MOVri | RID_EBP;
129 *(uint64_t *)p = (uint64_t)(g); p += 8;
130#else
131 *p++ = XI_MOVri | RID_EBP;
132 *(int32_t *)p = i32ptr(g); p += 4;
133#endif
134#if LJ_64
135 /* jmp [rip-pageofs] where lj_vm_ffi_callback is stored. */
136 *p++ = XI_GROUP5; *p++ = XM_OFS0 + (XOg_JMP<<3) + RID_EBP;
137 *(int32_t *)p = (int32_t)(page-(p+4)); p += 4;
138#else
139 /* jmp lj_vm_ffi_callback. */
140 *p++ = XI_JMP; *(int32_t *)p = target-(p+4); p += 4;
141#endif
142 } else {
143 *p++ = XI_JMPs; *p++ = (uint8_t)((2+2)*(31-(slot&31)) - 2);
144 }
145 }
146 return p;
147}
148#elif LJ_TARGET_ARM
149static void *callback_mcode_init(global_State *g, uint32_t *page)
150{
151 uint32_t *p = page;
152 void *target = (void *)lj_vm_ffi_callback;
153 MSize slot;
154 /* This must match with the saveregs macro in buildvm_arm.dasc. */
155 *p++ = ARMI_SUB|ARMF_D(RID_R12)|ARMF_N(RID_R12)|ARMF_M(RID_PC);
156 *p++ = ARMI_PUSH|ARMF_N(RID_SP)|RSET_RANGE(RID_R4,RID_R11+1)|RID2RSET(RID_LR);
157 *p++ = ARMI_SUB|ARMI_K12|ARMF_D(RID_R12)|ARMF_N(RID_R12)|CALLBACK_MCODE_HEAD;
158 *p++ = ARMI_STR|ARMI_LS_P|ARMI_LS_W|ARMF_D(RID_R12)|ARMF_N(RID_SP)|(CFRAME_SIZE-4*9);
159 *p++ = ARMI_LDR|ARMI_LS_P|ARMI_LS_U|ARMF_D(RID_R12)|ARMF_N(RID_PC);
160 *p++ = ARMI_LDR|ARMI_LS_P|ARMI_LS_U|ARMF_D(RID_PC)|ARMF_N(RID_PC);
161 *p++ = u32ptr(g);
162 *p++ = u32ptr(target);
163 for (slot = 0; slot < CALLBACK_MAX_SLOT; slot++) {
164 *p++ = ARMI_MOV|ARMF_D(RID_R12)|ARMF_M(RID_PC);
165 *p = ARMI_B | ((page-p-2) & 0x00ffffffu);
166 p++;
167 }
168 return p;
169}
170#elif LJ_TARGET_ARM64
171static void *callback_mcode_init(global_State *g, uint32_t *page)
172{
173 uint32_t *p = page;
174 void *target = (void *)lj_vm_ffi_callback;
175 MSize slot;
176 *p++ = A64I_LE(A64I_LDRLx | A64F_D(RID_X11) | A64F_S19(4));
177 *p++ = A64I_LE(A64I_LDRLx | A64F_D(RID_X10) | A64F_S19(5));
178 *p++ = A64I_LE(A64I_BR | A64F_N(RID_X11));
179 *p++ = A64I_LE(A64I_NOP);
180 ((void **)p)[0] = target;
181 ((void **)p)[1] = g;
182 p += 4;
183 for (slot = 0; slot < CALLBACK_MAX_SLOT; slot++) {
184 *p++ = A64I_LE(A64I_MOVZw | A64F_D(RID_X9) | A64F_U16(slot));
185 *p = A64I_LE(A64I_B | A64F_S26((page-p) & 0x03ffffffu));
186 p++;
187 }
188 return p;
189}
190#elif LJ_TARGET_PPC
191static void *callback_mcode_init(global_State *g, uint32_t *page)
192{
193 uint32_t *p = page;
194 void *target = (void *)lj_vm_ffi_callback;
195 MSize slot;
196 *p++ = PPCI_LIS | PPCF_T(RID_TMP) | (u32ptr(target) >> 16);
197 *p++ = PPCI_LIS | PPCF_T(RID_R12) | (u32ptr(g) >> 16);
198 *p++ = PPCI_ORI | PPCF_A(RID_TMP)|PPCF_T(RID_TMP) | (u32ptr(target) & 0xffff);
199 *p++ = PPCI_ORI | PPCF_A(RID_R12)|PPCF_T(RID_R12) | (u32ptr(g) & 0xffff);
200 *p++ = PPCI_MTCTR | PPCF_T(RID_TMP);
201 *p++ = PPCI_BCTR;
202 for (slot = 0; slot < CALLBACK_MAX_SLOT; slot++) {
203 *p++ = PPCI_LI | PPCF_T(RID_R11) | slot;
204 *p = PPCI_B | (((page-p) & 0x00ffffffu) << 2);
205 p++;
206 }
207 return p;
208}
209#elif LJ_TARGET_MIPS
210static void *callback_mcode_init(global_State *g, uint32_t *page)
211{
212 uint32_t *p = page;
213 uintptr_t target = (uintptr_t)(void *)lj_vm_ffi_callback;
214 uintptr_t ug = (uintptr_t)(void *)g;
215 MSize slot;
216#if LJ_TARGET_MIPS32
217 *p++ = MIPSI_LUI | MIPSF_T(RID_R3) | (target >> 16);
218 *p++ = MIPSI_LUI | MIPSF_T(RID_R2) | (ug >> 16);
219#else
220 *p++ = MIPSI_LUI | MIPSF_T(RID_R3) | (target >> 48);
221 *p++ = MIPSI_LUI | MIPSF_T(RID_R2) | (ug >> 48);
222 *p++ = MIPSI_ORI | MIPSF_T(RID_R3)|MIPSF_S(RID_R3) | ((target >> 32) & 0xffff);
223 *p++ = MIPSI_ORI | MIPSF_T(RID_R2)|MIPSF_S(RID_R2) | ((ug >> 32) & 0xffff);
224 *p++ = MIPSI_DSLL | MIPSF_D(RID_R3)|MIPSF_T(RID_R3) | MIPSF_A(16);
225 *p++ = MIPSI_DSLL | MIPSF_D(RID_R2)|MIPSF_T(RID_R2) | MIPSF_A(16);
226 *p++ = MIPSI_ORI | MIPSF_T(RID_R3)|MIPSF_S(RID_R3) | ((target >> 16) & 0xffff);
227 *p++ = MIPSI_ORI | MIPSF_T(RID_R2)|MIPSF_S(RID_R2) | ((ug >> 16) & 0xffff);
228 *p++ = MIPSI_DSLL | MIPSF_D(RID_R3)|MIPSF_T(RID_R3) | MIPSF_A(16);
229 *p++ = MIPSI_DSLL | MIPSF_D(RID_R2)|MIPSF_T(RID_R2) | MIPSF_A(16);
230#endif
231 *p++ = MIPSI_ORI | MIPSF_T(RID_R3)|MIPSF_S(RID_R3) | (target & 0xffff);
232 *p++ = MIPSI_JR | MIPSF_S(RID_R3);
233 *p++ = MIPSI_ORI | MIPSF_T(RID_R2)|MIPSF_S(RID_R2) | (ug & 0xffff);
234 for (slot = 0; slot < CALLBACK_MAX_SLOT; slot++) {
235 *p = MIPSI_B | ((page-p-1) & 0x0000ffffu);
236 p++;
237 *p++ = MIPSI_LI | MIPSF_T(RID_R1) | slot;
238 }
239 return p;
240}
241#else
242/* Missing support for this architecture. */
243#define callback_mcode_init(g, p) (p)
244#endif
245
246/* -- Machine code management --------------------------------------------- */
247
248#if LJ_TARGET_WINDOWS
249
250#define WIN32_LEAN_AND_MEAN
251#include <windows.h>
252
253#elif LJ_TARGET_POSIX
254
255#include <sys/mman.h>
256#ifndef MAP_ANONYMOUS
257#define MAP_ANONYMOUS MAP_ANON
258#endif
259
260#endif
261
262/* Allocate and initialize area for callback function pointers. */
263static void callback_mcode_new(CTState *cts)
264{
265 size_t sz = (size_t)CALLBACK_MCODE_SIZE;
266 void *p, *pe;
267 if (CALLBACK_MAX_SLOT == 0)
268 lj_err_caller(cts->L, LJ_ERR_FFI_CBACKOV);
269#if LJ_TARGET_WINDOWS
270 p = LJ_WIN_VALLOC(NULL, sz, MEM_RESERVE|MEM_COMMIT, PAGE_READWRITE);
271 if (!p)
272 lj_err_caller(cts->L, LJ_ERR_FFI_CBACKOV);
273#elif LJ_TARGET_POSIX
274 p = mmap(NULL, sz, (PROT_READ|PROT_WRITE), MAP_PRIVATE|MAP_ANONYMOUS,
275 -1, 0);
276 if (p == MAP_FAILED)
277 lj_err_caller(cts->L, LJ_ERR_FFI_CBACKOV);
278#else
279 /* Fallback allocator. Fails if memory is not executable by default. */
280 p = lj_mem_new(cts->L, sz);
281#endif
282 cts->cb.mcode = p;
283 pe = callback_mcode_init(cts->g, p);
284 UNUSED(pe);
285 lj_assertCTS((size_t)((char *)pe - (char *)p) <= sz,
286 "miscalculated CALLBACK_MAX_SLOT");
287 lj_mcode_sync(p, (char *)p + sz);
288#if LJ_TARGET_WINDOWS
289 {
290 DWORD oprot;
291 LJ_WIN_VPROTECT(p, sz, PAGE_EXECUTE_READ, &oprot);
292 }
293#elif LJ_TARGET_POSIX
294 mprotect(p, sz, (PROT_READ|PROT_EXEC));
295#endif
296}
297
298/* Free area for callback function pointers. */
299void lj_ccallback_mcode_free(CTState *cts)
300{
301 size_t sz = (size_t)CALLBACK_MCODE_SIZE;
302 void *p = cts->cb.mcode;
303 if (p == NULL) return;
304#if LJ_TARGET_WINDOWS
305 VirtualFree(p, 0, MEM_RELEASE);
306 UNUSED(sz);
307#elif LJ_TARGET_POSIX
308 munmap(p, sz);
309#else
310 lj_mem_free(cts->g, p, sz);
311#endif
312}
313
314/* -- C callback entry ---------------------------------------------------- */
315
316/* Target-specific handling of register arguments. Similar to lj_ccall.c. */
317#if LJ_TARGET_X86
318
319#define CALLBACK_HANDLE_REGARG \
320 if (!isfp) { /* Only non-FP values may be passed in registers. */ \
321 if (n > 1) { /* Anything > 32 bit is passed on the stack. */ \
322 if (!LJ_ABI_WIN) ngpr = maxgpr; /* Prevent reordering. */ \
323 } else if (ngpr + 1 <= maxgpr) { \
324 sp = &cts->cb.gpr[ngpr]; \
325 ngpr += n; \
326 goto done; \
327 } \
328 }
329
330#elif LJ_TARGET_X64 && LJ_ABI_WIN
331
332/* Windows/x64 argument registers are strictly positional (use ngpr). */
333#define CALLBACK_HANDLE_REGARG \
334 if (isfp) { \
335 if (ngpr < maxgpr) { sp = &cts->cb.fpr[ngpr++]; UNUSED(nfpr); goto done; } \
336 } else { \
337 if (ngpr < maxgpr) { sp = &cts->cb.gpr[ngpr++]; goto done; } \
338 }
339
340#elif LJ_TARGET_X64
341
342#define CALLBACK_HANDLE_REGARG \
343 if (isfp) { \
344 if (nfpr + n <= CCALL_NARG_FPR) { \
345 sp = &cts->cb.fpr[nfpr]; \
346 nfpr += n; \
347 goto done; \
348 } \
349 } else { \
350 if (ngpr + n <= maxgpr) { \
351 sp = &cts->cb.gpr[ngpr]; \
352 ngpr += n; \
353 goto done; \
354 } \
355 }
356
357#elif LJ_TARGET_ARM
358
359#if LJ_ABI_SOFTFP
360
361#define CALLBACK_HANDLE_REGARG_FP1 UNUSED(isfp);
362#define CALLBACK_HANDLE_REGARG_FP2
363
364#else
365
366#define CALLBACK_HANDLE_REGARG_FP1 \
367 if (isfp) { \
368 if (n == 1) { \
369 if (fprodd) { \
370 sp = &cts->cb.fpr[fprodd-1]; \
371 fprodd = 0; \
372 goto done; \
373 } else if (nfpr + 1 <= CCALL_NARG_FPR) { \
374 sp = &cts->cb.fpr[nfpr++]; \
375 fprodd = nfpr; \
376 goto done; \
377 } \
378 } else { \
379 if (nfpr + 1 <= CCALL_NARG_FPR) { \
380 sp = &cts->cb.fpr[nfpr++]; \
381 goto done; \
382 } \
383 } \
384 fprodd = 0; /* No reordering after the first FP value is on stack. */ \
385 } else {
386
387#define CALLBACK_HANDLE_REGARG_FP2 }
388
389#endif
390
391#define CALLBACK_HANDLE_REGARG \
392 CALLBACK_HANDLE_REGARG_FP1 \
393 if (n > 1) ngpr = (ngpr + 1u) & ~1u; /* Align to regpair. */ \
394 if (ngpr + n <= maxgpr) { \
395 sp = &cts->cb.gpr[ngpr]; \
396 ngpr += n; \
397 goto done; \
398 } CALLBACK_HANDLE_REGARG_FP2
399
400#elif LJ_TARGET_ARM64
401
402#define CALLBACK_HANDLE_REGARG \
403 if (isfp) { \
404 if (nfpr + n <= CCALL_NARG_FPR) { \
405 sp = &cts->cb.fpr[nfpr]; \
406 nfpr += n; \
407 goto done; \
408 } else { \
409 nfpr = CCALL_NARG_FPR; /* Prevent reordering. */ \
410 } \
411 } else { \
412 if (!LJ_TARGET_IOS && n > 1) \
413 ngpr = (ngpr + 1u) & ~1u; /* Align to regpair. */ \
414 if (ngpr + n <= maxgpr) { \
415 sp = &cts->cb.gpr[ngpr]; \
416 ngpr += n; \
417 goto done; \
418 } else { \
419 ngpr = CCALL_NARG_GPR; /* Prevent reordering. */ \
420 } \
421 }
422
423#elif LJ_TARGET_PPC
424
425#define CALLBACK_HANDLE_GPR \
426 if (n > 1) { \
427 lj_assertCTS(((LJ_ABI_SOFTFP && ctype_isnum(cta->info)) || /* double. */ \
428 ctype_isinteger(cta->info)) && n == 2, /* int64_t. */ \
429 "bad GPR type"); \
430 ngpr = (ngpr + 1u) & ~1u; /* Align int64_t to regpair. */ \
431 } \
432 if (ngpr + n <= maxgpr) { \
433 sp = &cts->cb.gpr[ngpr]; \
434 ngpr += n; \
435 goto done; \
436 }
437
438#if LJ_ABI_SOFTFP
439#define CALLBACK_HANDLE_REGARG \
440 CALLBACK_HANDLE_GPR \
441 UNUSED(isfp);
442#else
443#define CALLBACK_HANDLE_REGARG \
444 if (isfp) { \
445 if (nfpr + 1 <= CCALL_NARG_FPR) { \
446 sp = &cts->cb.fpr[nfpr++]; \
447 cta = ctype_get(cts, CTID_DOUBLE); /* FPRs always hold doubles. */ \
448 goto done; \
449 } \
450 } else { /* Try to pass argument in GPRs. */ \
451 CALLBACK_HANDLE_GPR \
452 }
453#endif
454
455#if !LJ_ABI_SOFTFP
456#define CALLBACK_HANDLE_RET \
457 if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
458 *(double *)dp = *(float *)dp; /* FPRs always hold doubles. */
459#endif
460
461#elif LJ_TARGET_MIPS32
462
463#define CALLBACK_HANDLE_GPR \
464 if (n > 1) ngpr = (ngpr + 1u) & ~1u; /* Align to regpair. */ \
465 if (ngpr + n <= maxgpr) { \
466 sp = &cts->cb.gpr[ngpr]; \
467 ngpr += n; \
468 goto done; \
469 }
470
471#if !LJ_ABI_SOFTFP /* MIPS32 hard-float */
472#define CALLBACK_HANDLE_REGARG \
473 if (isfp && nfpr < CCALL_NARG_FPR) { /* Try to pass argument in FPRs. */ \
474 sp = (void *)((uint8_t *)&cts->cb.fpr[nfpr] + ((LJ_BE && n==1) ? 4 : 0)); \
475 nfpr++; ngpr += n; \
476 goto done; \
477 } else { /* Try to pass argument in GPRs. */ \
478 nfpr = CCALL_NARG_FPR; \
479 CALLBACK_HANDLE_GPR \
480 }
481#else /* MIPS32 soft-float */
482#define CALLBACK_HANDLE_REGARG \
483 CALLBACK_HANDLE_GPR \
484 UNUSED(isfp);
485#endif
486
487#define CALLBACK_HANDLE_RET \
488 if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
489 ((float *)dp)[1] = *(float *)dp;
490
491#elif LJ_TARGET_MIPS64
492
493#if !LJ_ABI_SOFTFP /* MIPS64 hard-float */
494#define CALLBACK_HANDLE_REGARG \
495 if (ngpr + n <= maxgpr) { \
496 sp = isfp ? (void*) &cts->cb.fpr[ngpr] : (void*) &cts->cb.gpr[ngpr]; \
497 ngpr += n; \
498 goto done; \
499 }
500#else /* MIPS64 soft-float */
501#define CALLBACK_HANDLE_REGARG \
502 if (ngpr + n <= maxgpr) { \
503 UNUSED(isfp); \
504 sp = (void*) &cts->cb.gpr[ngpr]; \
505 ngpr += n; \
506 goto done; \
507 }
508#endif
509
510#define CALLBACK_HANDLE_RET \
511 if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
512 ((float *)dp)[1] = *(float *)dp;
513
514#else
515#error "Missing calling convention definitions for this architecture"
516#endif
517
518/* Convert and push callback arguments to Lua stack. */
519static void callback_conv_args(CTState *cts, lua_State *L)
520{
521 TValue *o = L->top;
522 intptr_t *stack = cts->cb.stack;
523 MSize slot = cts->cb.slot;
524 CTypeID id = 0, rid, fid;
525 int gcsteps = 0;
526 CType *ct;
527 GCfunc *fn;
528 int fntp;
529 MSize ngpr = 0, nsp = 0, maxgpr = CCALL_NARG_GPR;
530#if CCALL_NARG_FPR
531 MSize nfpr = 0;
532#if LJ_TARGET_ARM
533 MSize fprodd = 0;
534#endif
535#endif
536
537 if (slot < cts->cb.sizeid && (id = cts->cb.cbid[slot]) != 0) {
538 ct = ctype_get(cts, id);
539 rid = ctype_cid(ct->info); /* Return type. x86: +(spadj<<16). */
540 fn = funcV(lj_tab_getint(cts->miscmap, (int32_t)slot));
541 fntp = LJ_TFUNC;
542 } else { /* Must set up frame first, before throwing the error. */
543 ct = NULL;
544 rid = 0;
545 fn = (GCfunc *)L;
546 fntp = LJ_TTHREAD;
547 }
548 /* Continuation returns from callback. */
549 if (LJ_FR2) {
550 (o++)->u64 = LJ_CONT_FFI_CALLBACK;
551 (o++)->u64 = rid;
552 } else {
553 o->u32.lo = LJ_CONT_FFI_CALLBACK;
554 o->u32.hi = rid;
555 o++;
556 }
557 setframe_gc(o, obj2gco(fn), fntp);
558 if (LJ_FR2) o++;
559 setframe_ftsz(o, ((char *)(o+1) - (char *)L->base) + FRAME_CONT);
560 L->top = L->base = ++o;
561 if (!ct)
562 lj_err_caller(cts->L, LJ_ERR_FFI_BADCBACK);
563 if (isluafunc(fn))
564 setcframe_pc(L->cframe, proto_bc(funcproto(fn))+1);
565 lj_state_checkstack(L, LUA_MINSTACK); /* May throw. */
566 o = L->base; /* Might have been reallocated. */
567
568#if LJ_TARGET_X86
569 /* x86 has several different calling conventions. */
570 switch (ctype_cconv(ct->info)) {
571 case CTCC_FASTCALL: maxgpr = 2; break;
572 case CTCC_THISCALL: maxgpr = 1; break;
573 default: maxgpr = 0; break;
574 }
575#endif
576
577 fid = ct->sib;
578 while (fid) {
579 CType *ctf = ctype_get(cts, fid);
580 if (!ctype_isattrib(ctf->info)) {
581 CType *cta;
582 void *sp;
583 CTSize sz;
584 int isfp;
585 MSize n;
586 lj_assertCTS(ctype_isfield(ctf->info), "field expected");
587 cta = ctype_rawchild(cts, ctf);
588 isfp = ctype_isfp(cta->info);
589 sz = (cta->size + CTSIZE_PTR-1) & ~(CTSIZE_PTR-1);
590 n = sz / CTSIZE_PTR; /* Number of GPRs or stack slots needed. */
591
592 CALLBACK_HANDLE_REGARG /* Handle register arguments. */
593
594 /* Otherwise pass argument on stack. */
595 if (CCALL_ALIGN_STACKARG && LJ_32 && sz == 8)
596 nsp = (nsp + 1) & ~1u; /* Align 64 bit argument on stack. */
597 sp = &stack[nsp];
598 nsp += n;
599
600 done:
601 if (LJ_BE && cta->size < CTSIZE_PTR
602#if LJ_TARGET_MIPS64
603 && !(isfp && nsp)
604#endif
605 )
606 sp = (void *)((uint8_t *)sp + CTSIZE_PTR-cta->size);
607 gcsteps += lj_cconv_tv_ct(cts, cta, 0, o++, sp);
608 }
609 fid = ctf->sib;
610 }
611 L->top = o;
612#if LJ_TARGET_X86
613 /* Store stack adjustment for returns from non-cdecl callbacks. */
614 if (ctype_cconv(ct->info) != CTCC_CDECL) {
615#if LJ_FR2
616 (L->base-3)->u64 |= (nsp << (16+2));
617#else
618 (L->base-2)->u32.hi |= (nsp << (16+2));
619#endif
620 }
621#endif
622 while (gcsteps-- > 0)
623 lj_gc_check(L);
624}
625
626/* Convert Lua object to callback result. */
627static void callback_conv_result(CTState *cts, lua_State *L, TValue *o)
628{
629#if LJ_FR2
630 CType *ctr = ctype_raw(cts, (uint16_t)(L->base-3)->u64);
631#else
632 CType *ctr = ctype_raw(cts, (uint16_t)(L->base-2)->u32.hi);
633#endif
634#if LJ_TARGET_X86
635 cts->cb.gpr[2] = 0;
636#endif
637 if (!ctype_isvoid(ctr->info)) {
638 uint8_t *dp = (uint8_t *)&cts->cb.gpr[0];
639#if CCALL_NUM_FPR
640 if (ctype_isfp(ctr->info))
641 dp = (uint8_t *)&cts->cb.fpr[0];
642#endif
643#if LJ_TARGET_ARM64 && LJ_BE
644 if (ctype_isfp(ctr->info) && ctr->size == sizeof(float))
645 dp = (uint8_t *)&cts->cb.fpr[0].f[1];
646#endif
647 lj_cconv_ct_tv(cts, ctr, dp, o, 0);
648#ifdef CALLBACK_HANDLE_RET
649 CALLBACK_HANDLE_RET
650#endif
651 /* Extend returned integers to (at least) 32 bits. */
652 if (ctype_isinteger_or_bool(ctr->info) && ctr->size < 4) {
653 if (ctr->info & CTF_UNSIGNED)
654 *(uint32_t *)dp = ctr->size == 1 ? (uint32_t)*(uint8_t *)dp :
655 (uint32_t)*(uint16_t *)dp;
656 else
657 *(int32_t *)dp = ctr->size == 1 ? (int32_t)*(int8_t *)dp :
658 (int32_t)*(int16_t *)dp;
659 }
660#if LJ_TARGET_MIPS64 || (LJ_TARGET_ARM64 && LJ_BE)
661 /* Always sign-extend results to 64 bits. Even a soft-fp 'float'. */
662 if (ctr->size <= 4 &&
663 (LJ_ABI_SOFTFP || ctype_isinteger_or_bool(ctr->info)))
664 *(int64_t *)dp = (int64_t)*(int32_t *)dp;
665#endif
666#if LJ_TARGET_X86
667 if (ctype_isfp(ctr->info))
668 cts->cb.gpr[2] = ctr->size == sizeof(float) ? 1 : 2;
669#endif
670 }
671}
672
673/* Enter callback. */
674lua_State * LJ_FASTCALL lj_ccallback_enter(CTState *cts, void *cf)
675{
676 lua_State *L = cts->L;
677 global_State *g = cts->g;
678 lj_assertG(L != NULL, "uninitialized cts->L in callback");
679 if (tvref(g->jit_base)) {
680 setstrV(L, L->top++, lj_err_str(L, LJ_ERR_FFI_BADCBACK));
681 if (g->panic) g->panic(L);
682 exit(EXIT_FAILURE);
683 }
684 lj_trace_abort(g); /* Never record across callback. */
685 /* Setup C frame. */
686 cframe_prev(cf) = L->cframe;
687 setcframe_L(cf, L);
688 cframe_errfunc(cf) = -1;
689 cframe_nres(cf) = 0;
690 L->cframe = cf;
691 callback_conv_args(cts, L);
692 return L; /* Now call the function on this stack. */
693}
694
695/* Leave callback. */
696void LJ_FASTCALL lj_ccallback_leave(CTState *cts, TValue *o)
697{
698 lua_State *L = cts->L;
699 GCfunc *fn;
700 TValue *obase = L->base;
701 L->base = L->top; /* Keep continuation frame for throwing errors. */
702 if (o >= L->base) {
703 /* PC of RET* is lost. Point to last line for result conv. errors. */
704 fn = curr_func(L);
705 if (isluafunc(fn)) {
706 GCproto *pt = funcproto(fn);
707 setcframe_pc(L->cframe, proto_bc(pt)+pt->sizebc+1);
708 }
709 }
710 callback_conv_result(cts, L, o);
711 /* Finally drop C frame and continuation frame. */
712 L->top -= 2+2*LJ_FR2;
713 L->base = obase;
714 L->cframe = cframe_prev(L->cframe);
715 cts->cb.slot = 0; /* Blacklist C function that called the callback. */
716}
717
718/* -- C callback management ----------------------------------------------- */
719
720/* Get an unused slot in the callback slot table. */
721static MSize callback_slot_new(CTState *cts, CType *ct)
722{
723 CTypeID id = ctype_typeid(cts, ct);
724 CTypeID1 *cbid = cts->cb.cbid;
725 MSize top;
726 for (top = cts->cb.topid; top < cts->cb.sizeid; top++)
727 if (LJ_LIKELY(cbid[top] == 0))
728 goto found;
729#if CALLBACK_MAX_SLOT
730 if (top >= CALLBACK_MAX_SLOT)
731#endif
732 lj_err_caller(cts->L, LJ_ERR_FFI_CBACKOV);
733 if (!cts->cb.mcode)
734 callback_mcode_new(cts);
735 lj_mem_growvec(cts->L, cbid, cts->cb.sizeid, CALLBACK_MAX_SLOT, CTypeID1);
736 cts->cb.cbid = cbid;
737 memset(cbid+top, 0, (cts->cb.sizeid-top)*sizeof(CTypeID1));
738found:
739 cbid[top] = id;
740 cts->cb.topid = top+1;
741 return top;
742}
743
744/* Check for function pointer and supported argument/result types. */
745static CType *callback_checkfunc(CTState *cts, CType *ct)
746{
747 int narg = 0;
748 if (!ctype_isptr(ct->info) || (LJ_64 && ct->size != CTSIZE_PTR))
749 return NULL;
750 ct = ctype_rawchild(cts, ct);
751 if (ctype_isfunc(ct->info)) {
752 CType *ctr = ctype_rawchild(cts, ct);
753 CTypeID fid = ct->sib;
754 if (!(ctype_isvoid(ctr->info) || ctype_isenum(ctr->info) ||
755 ctype_isptr(ctr->info) || (ctype_isnum(ctr->info) && ctr->size <= 8)))
756 return NULL;
757 if ((ct->info & CTF_VARARG))
758 return NULL;
759 while (fid) {
760 CType *ctf = ctype_get(cts, fid);
761 if (!ctype_isattrib(ctf->info)) {
762 CType *cta;
763 lj_assertCTS(ctype_isfield(ctf->info), "field expected");
764 cta = ctype_rawchild(cts, ctf);
765 if (!(ctype_isenum(cta->info) || ctype_isptr(cta->info) ||
766 (ctype_isnum(cta->info) && cta->size <= 8)) ||
767 ++narg >= LUA_MINSTACK-3)
768 return NULL;
769 }
770 fid = ctf->sib;
771 }
772 return ct;
773 }
774 return NULL;
775}
776
777/* Create a new callback and return the callback function pointer. */
778void *lj_ccallback_new(CTState *cts, CType *ct, GCfunc *fn)
779{
780 ct = callback_checkfunc(cts, ct);
781 if (ct) {
782 MSize slot = callback_slot_new(cts, ct);
783 GCtab *t = cts->miscmap;
784 setfuncV(cts->L, lj_tab_setint(cts->L, t, (int32_t)slot), fn);
785 lj_gc_anybarriert(cts->L, t);
786 return callback_slot2ptr(cts, slot);
787 }
788 return NULL; /* Bad conversion. */
789}
790
791#endif
792