Tutorial 2: Reimplementing function calls

This tutorial assumes you have followed tutorial 1.

Step 1: rough reimplementation

In this tutorial, we reimplement OpenSHC::AI::AIVState::setupAIVMetadata. I have taken a decompilation from Ghidra and polished it with what we learned from Tutorial 1.

 1#include "OpenSHC/AI/AIVState.hpp"
 2#include "OpenSHC/Globals/DAT_AICState.hpp"
 3#include "OpenSHC/Globals/DAT_GameState.hpp"
 4
 5namespace OpenSHC {
 6namespace AI {
 7    
 8    // FUNCTION: STRONGHOLDCRUSADER 0x004ECEF0
 9    int AIVState::setupAIVMetadata(int playerID)
10
11    {
12        AIV* pAVar1;
13        int _buildIntervalUnk;
14        int _aivID;
15        int _aiType;
16
17        _aivID = 1;
18        pAVar1 = this->SEC_AIVS;
19        do {
20            pAVar1 = pAVar1 + 1;
21            if (pAVar1->playerID == 0) {
22                this->SEC_AIVS[_aivID].playerID = playerID;
23                _aiType = DAT_GameState::ptr->playerDataArray[playerID].aiType;
24                this->SEC_AIVS[_aivID].currentStepGoal = 0;
25                this->SEC_AIVS[_aivID].aivPoorCounter = 0;
26                this->SEC_AIVS[_aivID].aivSubType = 0;
27                this->SEC_AIVS[_aivID].aiType = _aiType;
28                _buildIntervalUnk = AICState::getAIBuildInterval(DAT_AICState::ptr, playerID);
29                this->SEC_AIVS[_aivID].aivPoorLimit_OR_AIC_buildInterval = _buildIntervalUnk;
30                this->aivCount = this->aivCount + 1;
31                return _aivID;
32            }
33            _aivID = _aivID + 1;
34        } while (_aivID < 9);
35        return 0;
36    }
37
38}
39}

Note that line 26 contains a method call in Ghidra style (pseudo C++). This style is not compatible with MSVC. Furthermore, it doesn’t take into account the OpenSHC approach of either calling the original function (if it hasn’t been fully reimplemented yet), or calling the reimplemented function (if it has been fully reimplemented).

Using the FunctionResolver to invoke member calls

We should change line 25 into:

_buildIntervalUnk = MACRO_CALL_MEMBER(AICState_Func::getAIBuildInterval, DAT_AICState::ptr)(playerID);

Step 2: comparing the reimplementation

We run the following to get a diff report:

reccmp/dll/run reccmp-reccmp --target STRONGHOLDCRUSADER --verbose 0x004ECEF0

 1
 2---
 3+++
 4@@ -0x4ecef0,38 +0x10030420,40 @@
 50x4ecef0 : push ebx     (setupAIVMetadata.cpp:12)
 60x4ecef1 : mov ebx, ecx
 70x4ecef3 : push edi
 80x4ecef4 : mov edi, 1   (setupAIVMetadata.cpp:18)
 90x4ecef9 : -lea eax, [ebx + 0x6d9c]
100x4eceff : -xor ecx, ecx
110x4ecf01 : -cmp dword ptr [eax], ecx
120x4ecf03 : -je 0x14
13           : +lea eax, [ebx + 4]        (setupAIVMetadata.cpp:19)
14           : +xor edx, edx
15           : +mov edi, edi
16           : +add eax, 0x6d98   (setupAIVMetadata.cpp:21)
17           : +cmp dword ptr [eax], edx  (setupAIVMetadata.cpp:22)
18           : +je 0xf
190x4ecf05 : add edi, 1   (setupAIVMetadata.cpp:34)
200x4ecf08 : -add eax, 0x6d98
210x4ecf0d : cmp edi, 9   (setupAIVMetadata.cpp:35)
220x4ecf10 : jl -0x11
230x4ecf12 : pop edi
240x4ecf13 : xor eax, eax         (setupAIVMetadata.cpp:36)
250x4ecf15 : pop ebx
260x4ecf16 : ret 4        (setupAIVMetadata.cpp:37)
270x4ecf19 : mov eax, edi         (setupAIVMetadata.cpp:23)
280x4ecf1b : imul eax, eax, 0x6d98
290x4ecf21 : push esi
300x4ecf22 : lea esi, [eax + ebx]
310x4ecf25 : mov eax, dword ptr [esp + 0x10]
320x4ecf29 : -mov edx, eax
330x4ecf2b : -imul edx, edx, 0x39f4
34           : +mov ecx, eax      (setupAIVMetadata.cpp:24)
35           : +imul ecx, ecx, 0x39f4
360x4ecf31 : mov dword ptr [esi + 4], eax
370x4ecf34 : -mov edx, dword ptr [edx + 0x115e0f8]
380x4ecf3a : -mov dword ptr [esi + 0x18], ecx
390x4ecf3d : -mov dword ptr [esi + 0x1c], ecx
400x4ecf40 : -mov dword ptr [esi + 0x14], ecx
41           : +mov ecx, dword ptr [ecx + 0x115e0f8]
42           : +mov dword ptr [esi + 8], ecx      (setupAIVMetadata.cpp:28)
430x4ecf43 : push eax     (setupAIVMetadata.cpp:29)
440x4ecf44 : mov ecx, 0x23fc8e8
450x4ecf49 : -mov dword ptr [esi + 8], edx
460x4ecf4c : -call <OFFSET1>
47           : +mov dword ptr [esi + 0x18], edx
48           : +mov dword ptr [esi + 0x1c], edx
49           : +mov dword ptr [esi + 0x14], edx
50           : +call FunctionResolver::Resolver<int (__thiscall OpenSHC::AI::AICState::*)(int),0,5026080,&OpenSHC::AI::AICState::getAIBuildInterval,0>::GameFunction<int (__thiscall OpenSHC::AI::AICState::*)(int)>::CallHelper<int,void>::call (FUNCTION)
510x4ecf51 : mov dword ptr [esi + 0x20], eax      (setupAIVMetadata.cpp:30)
520x4ecf54 : add dword ptr [ebx + 0xb6c68], 1     (setupAIVMetadata.cpp:31)
530x4ecf5b : pop esi
540x4ecf5c : mov eax, edi         (setupAIVMetadata.cpp:32)
550x4ecf5e : pop edi
560x4ecf5f : pop ebx
57           : +ret 4     (setupAIVMetadata.cpp:37)
58
59
60OpenSHC::AI::AIVState::setupAIVMetadata is only 64.10% similar to the original, diff above
61[OK] Virtual environment deactivated.
62[SUCCESS] Command completed successfully.

We achieved 64%!

Step 3: Dealing with non-reimplemented calls

Note line 50 which instead of calling an OFFSET, it calls FunctionResolver::Resolver<int (__thiscall OpenSHC::AI::AICState::*)(int),0,5026080,&OpenSHC::AI::AICState::getAIBuildInterval,0>::GameFunction<int (__thiscall OpenSHC::AI::AICState::*)(int)>::CallHelper<int,void>::call (FUNCTION).

This happens when the called function hasn’t been implemented yet. One solution is to temporarily stub a reimplementation for this function and set its reimplementation state to true. The other solution is to reimplement that function first.

For every reimplemented function to be called, the associated function resolver should have its reimplementation state set to true. So, for example, for function AICState::setFoodBuyPlan (see Tutorial 1) in file OpenSHC/AI/AICState.func.hpp we change line 2 below from , false, to true, . This detours all calls to setFoodBuyPlan through our DLL instead of through the original binary. Furthermore, functions calling setFoodBuyPlan have their percentages improved as the call points to the reimplementation.

1MACRO_FUNCTION_RESOLVER(
2    void (AICState::*)(int), false, Address::SHC_3BB0A8C1_0x004CB060, &AICState::setFoodBuyPlan)
3setFoodBuyPlan;

Step 4: reimplementing called functions

Let’s see whether we can reimplement that function while we are at it:

 1int AICState::getAIBuildInterval(int playerID)
 2{
 3  int _aiType;
 4  
 5  _aiType = DAT_GameState.playerDataArray[playerID].aiType;
 6  if (_aiType == 0) {
 7    return _aiType; // is the same as return 0;
 8  }
 9                    /* fixme: this accesses AIC.buildInterval !? */
10  return *(int *)((int)DAT_EntityState.seagullArray + _aiType * 0x2a4 + 0x24e8);
11}

Note that because _aiType is non-zero, the compiler optimized the access into the array by shifting the base, essentially mapping the [1-16] range unto a [0-15] array.

We can verify this by adding 0x2a4 to the offset of seagulArray + 0x24e8. We get the AIC buildInterval field of the zero entry.

Note that the raw assembly for that instruction says something entirely different, what we see in the decompilation is an artifact produced by Ghidra.

Luckily, this optimization is quite straightforward and always occurs, as this fix for the optimization achieves 100% accuracy with the original bytecode:

 1#include "OpenSHC/AI/AICState.hpp"
 2#include "OpenSHC/Globals/DAT_GameState.hpp"
 3namespace OpenSHC {
 4namespace AI {
 5
 6    // FUNCTION: STRONGHOLDCRUSADER 0x004cb120
 7    int AICState::getAIBuildInterval(int playerID)
 8
 9    {
10        int _aiType;
11
12        _aiType = DAT_GameState::ptr->playerDataArray[playerID].aiType;
13        if (_aiType == 0) {
14            return _aiType;
15        }
16
17        return this->DAT_AICArray[(_aiType - 1)].buildInterval;
18    }
19}
20}

Now let’s set the reimplemented status inside AICState.func.hpp to true:

1        MACRO_FUNCTION_RESOLVER(
2            int (AICState::*)(int), true, Address::SHC_3BB0A8C1_0x004CB120, &AICState::getAIBuildInterval)
3        getAIBuildInterval;

And rerun the comparison for setupAIVMetadata. Note that line 48 now corresponds with the correct function instead of a FunctionResolver statement.

 1---
 2+++
 3@@ -0x4ecef0,38 +0x1002ef40,40 @@
 40x4ecef0 : push ebx     (setupAIVMetadata.cpp:12)
 50x4ecef1 : mov ebx, ecx
 60x4ecef3 : push edi
 70x4ecef4 : mov edi, 1   (setupAIVMetadata.cpp:18)
 80x4ecef9 : -lea eax, [ebx + 0x6d9c]
 90x4eceff : -xor ecx, ecx
100x4ecf01 : -cmp dword ptr [eax], ecx
110x4ecf03 : -je 0x14
12           : +lea eax, [ebx + 4]        (setupAIVMetadata.cpp:19)
13           : +xor edx, edx
14           : +mov edi, edi
15           : +add eax, 0x6d98   (setupAIVMetadata.cpp:21)
16           : +cmp dword ptr [eax], edx  (setupAIVMetadata.cpp:22)
17           : +je 0xf
180x4ecf05 : add edi, 1   (setupAIVMetadata.cpp:34)
190x4ecf08 : -add eax, 0x6d98
200x4ecf0d : cmp edi, 9   (setupAIVMetadata.cpp:35)
210x4ecf10 : jl -0x11
220x4ecf12 : pop edi
230x4ecf13 : xor eax, eax         (setupAIVMetadata.cpp:36)
240x4ecf15 : pop ebx
250x4ecf16 : ret 4        (setupAIVMetadata.cpp:37)
260x4ecf19 : mov eax, edi         (setupAIVMetadata.cpp:23)
270x4ecf1b : imul eax, eax, 0x6d98
280x4ecf21 : push esi
290x4ecf22 : lea esi, [eax + ebx]
300x4ecf25 : mov eax, dword ptr [esp + 0x10]
310x4ecf29 : -mov edx, eax
320x4ecf2b : -imul edx, edx, 0x39f4
33           : +mov ecx, eax      (setupAIVMetadata.cpp:24)
34           : +imul ecx, ecx, 0x39f4
350x4ecf31 : mov dword ptr [esi + 4], eax
360x4ecf34 : -mov edx, dword ptr [edx + 0x115e0f8]
370x4ecf3a : -mov dword ptr [esi + 0x18], ecx
380x4ecf3d : -mov dword ptr [esi + 0x1c], ecx
390x4ecf40 : -mov dword ptr [esi + 0x14], ecx
40           : +mov ecx, dword ptr [ecx + 0x115e0f8]
41           : +mov dword ptr [esi + 8], ecx      (setupAIVMetadata.cpp:28)
420x4ecf43 : push eax     (setupAIVMetadata.cpp:29)
430x4ecf44 : mov ecx, 0x23fc8e8
440x4ecf49 : -mov dword ptr [esi + 8], edx
45           : +mov dword ptr [esi + 0x18], edx
46           : +mov dword ptr [esi + 0x1c], edx
47           : +mov dword ptr [esi + 0x14], edx
480x4ecf4c : call OpenSHC::AI::AICState::getAIBuildInterval (FUNCTION)
490x4ecf51 : mov dword ptr [esi + 0x20], eax      (setupAIVMetadata.cpp:30)
500x4ecf54 : add dword ptr [ebx + 0xb6c68], 1     (setupAIVMetadata.cpp:31)
510x4ecf5b : pop esi
520x4ecf5c : mov eax, edi         (setupAIVMetadata.cpp:32)
530x4ecf5e : pop edi
540x4ecf5f : pop ebx
55           : +ret 4     (setupAIVMetadata.cpp:37)
56
57
58OpenSHC::AI::AIVState::setupAIVMetadata is only 66.67% similar to the original, diff above

Step 5: improving accuracy using AI

Using Claude AI, the following solution gives a special 100% match:

 1#include "OpenSHC/AI/AICState.func.hpp"
 2#include "OpenSHC/AI/AIVState.hpp"
 3#include "OpenSHC/Globals/DAT_AICState.hpp"
 4#include "OpenSHC/Globals/DAT_GameState.hpp"
 5
 6namespace OpenSHC {
 7namespace AI {
 8
 9    // FUNCTION: STRONGHOLDCRUSADER 0x004ECEF0
10    int AIVState::setupAIVMetadata(int playerID)
11    {
12        int aivID = 1;
13        AIVSpec* pSlot = &this->SEC_AIVS[1];
14        do {
15            if (pSlot->playerID == 0) {
16                this->SEC_AIVS[aivID].playerID = playerID;
17                this->SEC_AIVS[aivID].aivPoorCounter = 0;
18
19                int aiType = DAT_GameState::ptr->playerDataArray[playerID].aiType;
20                this->SEC_AIVS[aivID].aiType = aiType;
21
22                this->SEC_AIVS[aivID].aivSubType = 0;
23                this->SEC_AIVS[aivID].currentStepGoal = 0;
24
25                this->SEC_AIVS[aivID].aivPoorLimit_OR_AIC_buildInterval
26                    = MACRO_CALL_MEMBER(AICState_Func::getAIBuildInterval, DAT_AICState::ptr)(playerID);
27
28                this->aivCount = this->aivCount + 1;
29
30                return aivID;
31            }
32            aivID = aivID + 1;
33            pSlot = pSlot + 1;
34        } while (aivID < 9);
35        return 0;
36    }
37
38}
39}

Effective matches

The reccmp output indicates a special form of 100% accuracy:


0x4ecef0: OpenSHC::AI::AIVState::setupAIVMetadata 100% effective match (differs, but only in ways that don't affect behavior).

✨ OK! ✨

Achieving a true 100% match

It turns out some code lines can be reordered to achieve a full 100% match. If you are a perfectionist, go ahead!

This is the right order:

 1this->SEC_AIVS[aivID].playerID = playerID;
 2
 3int aiType = DAT_GameState::ptr->playerDataArray[playerID].aiType;
 4this->SEC_AIVS[aivID].aiType = aiType;
 5
 6this->SEC_AIVS[aivID].currentStepGoal = 0;
 7this->SEC_AIVS[aivID].aivPoorCounter = 0;
 8this->SEC_AIVS[aivID].aivSubType = 0;
 9
10this->SEC_AIVS[aivID].aivPoorLimit_OR_AIC_buildInterval
11    = MACRO_CALL_MEMBER(AICState_Func::getAIBuildInterval, DAT_AICState::ptr)(playerID);
12
13this->aivCount = this->aivCount + 1;

Step 6: Pull Request preparation

See Tutorial 1. Make sure to only submit the .cpp file and status/ update for one function per PR or multiple functions that belong logically together (such as class methods of the same class).

Note: reimplementation of library functions

Not all called functions can reimplemented, as some are windows library functions that were statically included into the binary. Note that not all library functions start with _.

Windows library functions

For example, line 20 calls the windows _rand() function:

 1#include "OpenSHC/Global.func.hpp"
 2#include "OpenSHC/OS.func.hpp"
 3#include "OpenSHC/Random/RNG.func.hpp"
 4#include "OpenSHC/Random/RNG.hpp"
 5
 6namespace OpenSHC {
 7namespace Random {
 8
 9    // FUNCTION: STRONGHOLDCRUSADER 0x0046a760
10    void RNG::populateRNG1040()
11
12    {
13
14        MACRO_CALL(OpenSHC::Global_Func::SetRNGSeed)(this->seed);
15        this->index2 = 0;
16        this->index1 = 0;
17        short* _pRandomNumber = &this->randomNumbers[0];
18        int n = 20000;
19        do {
20            int random = MACRO_CALL(OpenSHC::OS_Func::_rand)();
21            *_pRandomNumber = (short)random;
22            _pRandomNumber += 1;
23            n -= 1;
24        } while (n != 0);
25
26        this->currentNumber2 = this->randomNumbers[this->index2];
27        this->index2 += 1;
28
29        this->currentNumber1 = this->randomNumbers[this->index1];
30        this->index1 += 1;
31
32        return;
33    }
34
35}
36}

Compiler-injected functions

For math operations that involve floating point numbers, functions are injected as well. See line 3 and line 6 below (OpenSHC::IO::FilePackager::writeMapOrSaveFile):

 1        if (local_8 != (RenderLoadAndSaveBar *)0x0) {
 2          _someBigNumbeKeeper =
 3               __allmul(_secSizeSumUnk,(int)_secSizeSumUnk >> 0x1f,1000,0);
 4          mapSectionAddressArray = (MapSectionAddress *)(_someBigNumbeKeeper >> 0x20);
 5          _someRelativeSizeValue =
 6               __alldiv
 7                         ((long)_someBigNumbeKeeper,(long)mapSectionAddressArray,_rawSize,
 8                          _rawSize >> 0x1f);
 9          (*local_8)((int)_someRelativeSizeValue);
10        }

As this injection of functions cannot be replicated from source code, we ignore these in reimplementations and hope the compiler injects them again.

You won’t see such functions listed in the OpenSHC::OS namespace.

Note: ignoring Ghidra function sugar

Ghidra includes helper functions to explicate what is happening logically. One such example is the ADJ() function, which represents relative offsets from one field to another. Note how the ptr on line 12 is the input of ADJ() to get access to other fields.

Ghida functions like this are not reimplemented and can be removed/ignored.

 1int __thiscall
 2OpenSHC::Map::Units::UnitsState::getAliveLordForPlayer(UnitsState *this,int playerID)
 3
 4{
 5  int _unit;
 6  Unit * 150 _ptrUnit;
 7  
 8  _unit = 1;
 9  if (1 < (int)DAT_UnitsState.maxUnitCount) {
10                    /* Unit offset 150 */
11    _ptrUnit = &DAT_UnitsState.units[1].owner;
12    do {
13      if ((((ADJ(_ptrUnit)->unitType == UT_LORD) && (ADJ(_ptrUnit)->owner == playerID)) &&
14          (ADJ(_ptrUnit)->logicalState == ULS_NORMAL)) && (ADJ(_ptrUnit)->dying == 0)) {
15        return _unit;
16      }
17      _unit = _unit + 1;
18      _ptrUnit = _ptrUnit + 0x248;
19    } while (_unit < (int)DAT_UnitsState.maxUnitCount);
20  }
21  return 0;
22}