Three months ago I posted here about using your own custom stack for one invocation of a function:

https://groups.google.com/g/comp.lang.c++/c/m1uMfMv1fjE

In my 2nd post in that thread, I share code that also supports exception handling:

https://godbolt.org/z/M9544Meqq

Up until now though, I've only had this working with very simple functions that have the signature:

void Func(void);

Today I've written it to support functions that have any signature (any number of parameters, and also a return value).

I start off with three thread_local global variables:

thread_local char *p_original, *p_replacement;
thread_local void (*f)(void);

'p_original' will hold the address of the original stack
'p_replacement' will hold the address of the replacement stack
'f' will hold the address of the function to be invoked

Accessing thread_local variables from assembler can be a little difficult (particularly when writing inline assembler inside a C++ source file), and so to make this a little easier, I've written four functions in C++ to get/set the thread_local variables for me:

char *Get_p_original(void) { return p_original; }
void Set_p_original(char *const arg) { p_original = arg; }
char *Get_p_replacement(void) { return p_replacement; }
void (*Get_f(void))(void) { return f; }

The x86_64 assembler for the getter functions will put the address inside the 'RAX' register. For example the assembler for "Get_f" will simply be:

mov rax, fs:f
ret

The 'set' function will expect the sole function argument to be inside the RDI register on Linux (or inside the RCX register on Microsoft Windows).

I'm going to write a function in Assembler called 'set_stack_pointer_and_invoke', the purpose of which is to change the stack pointer, invoke the function, then restore the original stack pointer. Note that the stack must be intact inside this function before I invoke the target function (so if I push anything onto the stack, I must pop it off the stack before invoking the target function). Here's the inline assembler for the GNU compiler:

__asm("Assembler_set_stack_pointer_and_invoke:\n"
".intel_syntax noprefix \n"
// Step 1: Save the original stack pointer
" push rdi \n" // Save 'rdi' to restore it later
" mov rdi, rsp \n"
" add rdi, 8 \n" // Stack pointer value before we pushed 'rdi'
" call Set_p_original \n" // sets p_original to 'rdi'
" pop rdi \n"
// Step 2: Retrieve the replacement stack pointer
" call Get_p_replacement \n" // sets 'rax' to p_replacement
" mov rsp, rax \n"
// Step 3: Retrieve the address of the function
" call Get_f \n" // sets 'rax' to 'f'
// Step 4: Invoke the function
" call rax \n" // --- Invoke the function!
// Step 5: Save the return value
" push rax \n" // keep track of lower 64 bits of the return value
" push rdx \n" // keep track of higher 64 bits of the return value
// Step 6: Retrieve the original stack pointer
" call Get_p_original \n" // sets 'rax' to p_original
" mov rcx, rax \n" // 'rcx' is caller-saved on Linux, so we can clobber it
// Step 7: Restore the return value
" pop rdx \n"
" pop rax \n"
// Step 8: Restore the original stack pointer
" mov rsp, rcx \n"
" ret \n" // return
".att_syntax");

So next what I did was write a class called 'Stacker' which manages the new custom stack:

class Stacker {
char *p;
std::unique_ptr<char[]> mystack;

public:

Stacker(std::size_t const len) noexcept(false) // might throw bad_alloc
{
assert( len >= 128u );

mystack.reset( new char[len] );
p = mystack.get() + len - 16u;
}

Stacker(char *const arg, std::size_t const len) noexcept
{
assert( nullptr != arg );
assert( len >= 128u );

p = arg + len - 16u;
}

template<typename R, typename... Params>
Invoker<R,Params...> operator()( R(*const arg)(Params...) )
{
return Invoker<R,Params...>(this->p, arg);
}
};

So you create a 'Stacker' object with the size of the stack you want, and you use 'operator()' to give it the address of the function you want to invoke, so you use it as follows:

Stacker(1048576000u)(MyTargetFunction)

The 'operator()' returns an object of type 'Invoker', which is defined as follows:

template<typename R, typename... Params>
class Invoker {
public:
Invoker(char *const arg_p, R(*const arg_f)(Params...))
{
p_replacement = arg_p; // sets a thread_local variable
f = reinterpret_cast<void (*)(void)>(arg_f); // sets a thread_local variable
}

R operator()(Params... args) // This could be static function but I like operator()
{
R (*const funcptr)(Params...) = reinterpret_cast<R(*)(Params....)>(Assembler_set_stack_pointer_and_invoke);
return funcptr(args...);
}
};

And so let's say we want to invoke the following function:

int Func2(double a, float b, short c)
{
return a + b + c;
}

So then we would use the Stacker as follows:

int retval = Stacker(1048576000u)(Func2)(2.0, 3.0f, 8);

A simplified version of the code is up on GodBolt:

https://godbolt.org/z/rTvqs4jvY

The only limitation of this technique is that the return value must be <= 16 bytes (because the lower 64 bits go in RAX, and the upper 64 bits go in RDX). Where the return value is > 16 bytes, the return value is pushed onto the stack -- and I haven't coded the solution for that yet (it will be tricky).

And here's the full code copy-pasted:

#include <cassert> // assert
#include <cstddef> // size_t
#include <memory> // unique_ptr

inline static thread_local char *p_original, *p_replacement;
inline static thread_local void (*f)(void);

extern "C" {
void Assembler_set_stack_pointer_and_invoke(void) noexcept;

char *Get_p_original(void) { return p_original; }
void Set_p_original(char *const arg) { p_original = arg; }
char *Get_p_replacement(void) { return p_replacement; }
void (*Get_f(void))(void) { return f; }
}

template<typename R, typename... Params>
class Invoker {
public:
Invoker(char *const arg_p, R(*const arg_f)(Params...))
{
p_replacement = arg_p; // sets a thread_local variable
f = reinterpret_cast<void (*)(void)>(arg_f); // sets a thread_local variable
}

R operator()(Params... args) // This could be static function but I like operator()
{
R (*const funcptr)(Params...) = reinterpret_cast<R(*)(Params...)>(Assembler_set_stack_pointer_and_invoke);
return funcptr(args...);
}
};

class Stacker {
char *p;
std::unique_ptr<char[]> mystack;

public:

Stacker(std::size_t const len) noexcept(false) // might throw bad_alloc
{
assert( len >= 128u );

mystack.reset( new char[len] );
p = mystack.get() + len - 16u;
}

Stacker(char *const arg, std::size_t const len) noexcept
{
assert( nullptr != arg );
assert( len >= 128u );

p = arg + len - 16u;
}

template<typename R, typename... Params>
Invoker<R,Params...> operator()( R(*const arg)(Params...) )
{
return Invoker<R,Params...>(this->p, arg);
}
};

__asm("Assembler_set_stack_pointer_and_invoke:\n"
".intel_syntax noprefix \n"
// Step 1: Save the original stack pointer
" push rdi \n" // Save 'rdi' to restore it later
" mov rdi, rsp \n"
" add rdi, 8 \n" // Stack pointer value before we pushed 'rdi'
" call Set_p_original \n" // sets p_original to 'rdi'
" pop rdi \n"
// Step 2: Retrieve the replacement stack pointer
" call Get_p_replacement \n" // sets 'rax' to p_replacement
" mov rsp, rax \n"
// Step 3: Retrieve the address of the function
" call Get_f \n" // sets 'rax' to 'f'
// Step 4: Invoke the function
" call rax \n" // --- Invoke the function!
// Step 5: Save the return value
" push rax \n" // keep track of lower 64 bits of the return value
" push rdx \n" // keep track of higher 64 bits of the return value
// Step 6: Retrieve the original stack pointer
" call Get_p_original \n" // sets 'rax' to p_original
" mov rcx, rax \n" // 'rcx' is caller-saved on Linux, so we can clobber it
// Step 7: Restore the return value
" pop rdx \n"
" pop rax \n"
// Step 8: Restore the original stack pointer
" mov rsp, rcx \n"
" ret \n" // return
".att_syntax");

// =================== And now the test code ==============================================
#include <iostream> // cout, endl
using std::cout, std::endl;

void Recursive(unsigned const arg)
{ if ( 0u == arg ) return;
cout << arg << endl;
Recursive(arg - 1u);
}

void Func1(void)
{ Recursive(2000000u); // This segfaults with normal stack
}

int Func2(double a, float b, short c)
{ return a + b + c;
}

int main(void)
{ cout << "first line in main\n";

Stacker(1048576000u)(Func1)();
cout << "Retval: " << Stacker(1048576000u)(Func2)(2.0, 3.0f, 8) << endl;

cout << "last line in main\n";
}