Simple Encryptor reverse engineering challenge Writeup

This is a writeup of Simple Encryptor reverse engineering challenge in HackTheBox.

Disassemble the code
Exploit

Disassemble the code

After downloading and unzipping a zip file, I obtained two files: encrypt and flag.enc.

The encrypt file is an ELF (Executable and Linkable Format) file, while flag.enc is a file that contains some binary data. I suspect that the encrypt file was used to encrypt the flag, which was then stored in flag.enc.

└─[$] file encrypt
encrypt: ELF 64-bit LSB pie executable, x86-64, version 1 (SYSV), dynamically linked, interpreter /lib64/ld-linux-x86-64.so.2, BuildID[sha1]=0bddc0a794eca6f6e2e9dac0b6190b62f07c4c75, for GNU/Linux 3.2.0, not stripped

└─[$] file flag.enc
flag.enc: data
└─[$] cat flag.enc
Z5�b�>�����u���9�K�!�C#qe�'K%

Let’s disassemble encrypt. Upon using Ghidra to disassemble encrypt, I obtained the following code.

undefined8 main(void)

{
  int iVar1;
  time_t tVar2;
  long in_FS_OFFSET;
  uint local_40;
  uint local_3c;
  long local_38;
  FILE *local_30;
  size_t local_28;
  void *local_20;
  FILE *local_18;
  long local_10;
  
  local_10 = *(long *)(in_FS_OFFSET + 0x28);
  local_30 = fopen("flag","rb");
  fseek(local_30,0,2);
  local_28 = ftell(local_30);
  fseek(local_30,0,0);
  local_20 = malloc(local_28);
  fread(local_20,local_28,1,local_30);
  fclose(local_30);
  tVar2 = time((time_t *)0x0);
  local_40 = (uint)tVar2;
  srand(local_40);
  for (local_38 = 0; local_38 < (long)local_28; local_38 = local_38 + 1) {
    iVar1 = rand();
    *(byte *)((long)local_20 + local_38) = *(byte *)((long)local_20 + local_38) ^ (byte)iVar1;
    local_3c = rand();
    local_3c = local_3c & 7;
    *(byte *)((long)local_20 + local_38) =
         *(byte *)((long)local_20 + local_38) << (sbyte)local_3c |
         *(byte *)((long)local_20 + local_38) >> 8 - (sbyte)local_3c;
  }
  local_18 = fopen("flag.enc","wb");
  fwrite(&local_40,1,4,local_18);
  fwrite(local_20,1,local_28,local_18);
  fclose(local_18);
  if (local_10 != *(long *)(in_FS_OFFSET + 0x28)) {
                    /* WARNING: Subroutine does not return */
    __stack_chk_fail();
  }
  return 0;
}

These varible names are not easy to read. So, I changed the variable name and add some comments to understand the code easier. Basically, the code consist of these following steps.

It reads the flag into a variable called ‘flag’.
It initializes ‘srand’ with a random seed value.
It uses the generated random values to encrypt the flag.
It stores the first four bytes of the seed, along with the encrypted flag, into the flag.enc file.

undefined8 main(void)

{
  int rnd1;
  time_t tVar1;
  long in_FS_OFFSET;
  uint seed;
  uint rnd2;
  long i;
  FILE *flagFile;
  size_t flagSize;
  void *flag;
  FILE *encFlagFile;
  long fs;
  
  fs = *(long *)(in_FS_OFFSET + 0x28);

  // read flag into flag variable
  flagFile = fopen("flag","rb");
  fseek(flagFile,0,2);
  flagSize = ftell(flagFile);
  fseek(flagFile,0,0);
  flag = malloc(flagSize);
  fread(flag,flagSize,1,flagFile);
  fclose(flagFile);

  // initialize srand with the seed
  tVar1 = time((time_t *)0x0);
  seed = (uint)tVar1;
  srand(seed);

  // encryption process
  for (i = 0; i < (long)flagSize; i = i + 1) {
    rnd1 = rand();
    *(byte *)((long)flag + i) = *(byte *)((long)flag + i) ^ (byte)rnd1;
    rnd2 = rand();
    rnd2 = rnd2 & 7;
    *(byte *)((long)flag + i) =
         *(byte *)((long)flag + i) << (sbyte)rnd2 | *(byte *)((long)flag + i) >> 8 - (sbyte)rnd2;
  }

  encFlagFile = fopen("flag.enc","wb");
  // store the first 4 bytes of the seed
  fwrite(&seed,1,4,encFlagFile);
  // store the encrypted flag
  fwrite(flag,1,flagSize,encFlagFile);
  fclose(encFlagFile);

  if (fs != *(long *)(in_FS_OFFSET + 0x28)) {
                    /* WARNING: Subroutine does not return */
    __stack_chk_fail();
  }
  return 0;
}

The difficult point is the random values used for encryption, generated by the rand() function based on a random seed value. However, the code where it stores the encrypted flag into flag.enc, stores also the first 4 bytes of the seed.

encFlagFile = fopen("flag.enc","wb");
// store the first 4 bytes of the seed
fwrite(&seed,1,4,encFlagFile);
// store the encrypted flag
fwrite(flag,1,flagSize,encFlagFile);

Now I need to understand how the seed and rand() function work.

According to the link,

The seed value for the rand function. The seed value determines a particular sequence of random numbers when calling the rand function. If a program always uses the same seed value, the rand function will always get the same sequence of numbers.

Consequently, by using the same seed, we can reproduce the identical random values that were used in the encryption loop.

To verify the accuracy of this logic, I’ve crafted the following code.

#include <stdio.h>
#include <stdlib.h>

int main(void) {
    FILE *f;
    size_t flagSize;
    unsigned int seed;
    f = fopen("flag.enc", "rb");
    // seek until the end of the file to get the size
    fseek(f, 0, SEEK_END);
    flagSize = ftell(f);
    // seek to the beginning
    fseek(f, 0, SEEK_SET);

    fread(&seed, 1, 4, f);
    fclose(f);

    printf("seed: %d\n", seed);
    srand(seed);

    for(int i = 0; i < (long)flagSize; i++) {
        printf("%d\n", rand());
    }

    return 0;
}

Every time I executed the poc, the same output was generated.

└─[$] gcc poc.c -o poc
└─[$] ./poc
seed: 1655780698
2014390344
1469870660
1982650619
1625766133
692418227
51836669
1207566194
29148633
1814471238
...

The final aspect to tackle is the encryption process, as depicted below:

rnd1 = rand()
flag = flag ^ rnd1
rnd2 = rand()
flagEnc = flag << rnd2 | flag >> 8 - rnd2

To reverse this process, the logic would be as follows:

rnd1 = rand()
rnd2 = rand()
flagEnc = flagEnc >> rnd2 | flagEnc << 8 - rnd2
flag = flagEnc ^ rnd1

Exploit

Now it’s time to exploit it. Below is my exploit code.

#include <stdio.h>
#include <stdlib.h>

int main(void) {
    typedef unsigned char byte;
    FILE *f;
    size_t flagSize;
    byte *flag;
    unsigned int seed;
    long i;
    int rnd1, rnd2;

    f = fopen("flag.enc", "rb");
    // seek until the end of the file to get the size
    fseek(f, 0, SEEK_END);
    flagSize = ftell(f);
    // seek to the beginning
    fseek(f, 0, SEEK_SET);
    // allocate memory of the flag
    flag = malloc(flagSize);
    fread(flag, 1, flagSize, f);
    fclose(f);

    // take seed from the first 4 bytes
    int flagOffset = 4;
    memcpy(&seed, flag, flagOffset);
    srand(seed);

    for(i = flagOffset; i < (long)flagSize; i++) {
        rnd1 = rand();
        rnd2 = rand();
        rnd2 = rnd2 & 7;
        flag[i]  = 
            flag[i] >> rnd2 |
            flag[i] << 8 - rnd2;
        flag[i] = rnd1 ^ flag[i];
        printf("%c", flag[i]);
    }
}

└─[$] gcc exploit.c -o exploit
└─[$] ./exploit

Flag: xxxxxxxxxxxxxx

Published Jul 30, 2023

Backend engineer, living in BerlinHiroki Fujino on Twitter