Simple Encryptor
Simple Encryptor
Understanding what’s happening
We get two files after extraction:
1
2
-rwxr-xr-x 1 kali kali 17120 Jul 19 2022 encrypt
-rw-r--r-- 1 kali kali 32 Jul 19 2022 flag.enc
Let’s understand what the encrypt file is:
1
2
file encrypt
encrypt: ELF 64-bit LSB pie executable, x86-64, version 1 (SYSV), dynamically linked, interpreter /lib64/ld-linux-x86-64.so.2, BuildID[sha1]=0bddc0a794eca6f6e2e9dac0b6190b62f07c4c75, for GNU/Linux 3.2.0, not stripped
Let’s try to run it:
1
2
./encrypt
zsh: segmentation fault ./encrypt
We are getting a seg fault. Let’s ltrace it:
1
2
3
4
5
ltrace ./encrypt
fopen("flag", "rb") = nil
fseek(nil, 0, SEEK_END <no return ...>
--- SIGSEGV (Segmentation fault) ---
+++ killed by SIGSEGV +++
It is trying to open a file named “flag”, but since it doesn’t exist we get a seg fault. Let’s create a file “flag” and ltrace the binary again:
ltrace
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
echo 'HTB{a_flag}' > flag
ltrace ./encrypt
fopen("flag", "rb") = 0x55ba679312a0
fseek(0x55ba679312a0, 0, SEEK_END) = 0
ftell(0x55ba679312a0) = 12
fseek(0x55ba679312a0, 0, SEEK_SET) = 0
malloc(12) = 0x55ba67932490
fread(0x55ba67932490, 12, 1, 0x55ba679312a0) = 1
fclose(0x55ba679312a0) = 0
time(nil) = 1756461975
srand(1756461975) = <void>
rand() = 966692124
rand() = 1934908113
rand() = 571083816
rand() = 1788839109
rand() = 170686150
rand() = 800678257
rand() = 1400291990
rand() = 129862145
rand() = 1392776060
rand() = 702775223
rand() = 1658851545
rand() = 662877018
rand() = 1077806250
rand() = 1710868276
rand() = 34039147
rand() = 1339006387
rand() = 818123230
rand() = 867934306
rand() = 1091013604
rand() = 1336249483
rand() = 1780134332
rand() = 278967634
rand() = 575990981
rand() = 1287463525
fopen("flag.enc", "wb") = 0x55ba679312a0
fwrite("\227{\261h\005", 1, 4, 0x55ba679312a0) = 4
fwrite("\250\217\t\333\216\032\3148\376\034\a\371", 1, 12, 0x55ba679312a0) = 12
fclose(0x55ba679312a0) = 0
+++ exited (status 0) +++
What actually happens:
We see the binary opens our “flag” file.
- The file pointer is set to the end of the file.
- Then the position of file pointer is return. (The value returned is 12 bytes)
- Then file pointer is set to beginning of file.
- Creates a buffer size of our file.
- Copies the content of file into the buffer.
- File is closed.
- Gets the current timestamp.
- Seeds the
srand()function with the timestamp. rand()function is called 24 times.- Writes the encrypted data to
flag.enc
Decompiling main()
Throwing the binary in Ghidra gets us the following decompiled main():
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
undefined8 main(void)
{
int iVar1;
time_t tVar2;
long in_FS_OFFSET;
uint local_40;
uint local_3c;
long local_38;
FILE *local_30;
size_t local_28;
void *local_20;
FILE *local_18;
long local_10;
local_10 = *(long *)(in_FS_OFFSET + 0x28);
local_30 = fopen("flag","rb");
fseek(local_30,0,2);
local_28 = ftell(local_30);
fseek(local_30,0,0);
local_20 = malloc(local_28);
fread(local_20,local_28,1,local_30);
fclose(local_30);
tVar2 = time((time_t *)0x0);
local_40 = (uint)tVar2;
srand(local_40);
for (local_38 = 0; local_38 < (long)local_28; local_38 = local_38 + 1) {
iVar1 = rand();
*(byte *)((long)local_20 + local_38) = *(byte *)((long)local_20 + local_38) ^ (byte)iVar1;
local_3c = rand();
local_3c = local_3c & 7;
*(byte *)((long)local_20 + local_38) =
*(byte *)((long)local_20 + local_38) << (sbyte)local_3c |
*(byte *)((long)local_20 + local_38) >> 8 - (sbyte)local_3c;
}
local_18 = fopen("flag.enc","wb");
fwrite(&local_40,1,4,local_18);
fwrite(local_20,1,local_28,local_18);
fclose(local_18);
if (local_10 != *(long *)(in_FS_OFFSET + 0x28)) {
/* WARNING: Subroutine does not return */
__stack_chk_fail();
}
return 0;
}
What we care about (rest we figured from ltrace):
1
2
3
4
5
6
7
iVar1 = rand(); // Get random number
byte ^= (byte)iVar1; // XOR with low byte of random number
local_3c = rand(); // Get another random number
local_3c = local_3c & 7; // Keep only lower 3 bits (0-7)
// Rotate the byte left by (local_3c) positions:
byte = (byte << local_3c) | (byte >> (8 - local_3c));
1
2
iVar1 = rand(); // Returns 32-bit int (e.g., 0x12345678)
byte ^= (byte)iVar1; // Cast to byte takes only lowest 8 bits
rand() returns: 0x12345678 (32 bits) (byte)iVar1 gives: 0x78 (only the rightmost 8 bits)
1
local_3c = rand() & 7;
7 in binary is 00000111 ANDing with 111 keeps only the 3 rightmost bits This gives values 0-7 (perfect for bit rotation amounts)
1
byte = (byte << local_3c) | (byte >> (8 - local_3c));
This is a circular left shift. Let’s say byte = 0b11010000 and local_3c = 3:
byte << 3: Shift left 3 positions: 0b10000000 (lost bits: 110) byte >> (8-3): Shift right 5 positions: 0b00000110 (brings back lost bits) | (OR): Combines them: 0b10000110
The lost bits from the left shift “wrap around” to the right side.
Getting the timestamp
For now we care about getting the timestamp used to encrypt our flag:
1
2
xxd -l 4 flag.enc
00000000: 5a35 b162 Z5.b
Decryption
We will reverse the order.
We will first right rotate then XOR, but we will have to use the same timestamp:
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
#!/usr/bin/env python3
import struct
import ctypes
def decrypt_flag(filename):
"""
Decrypts the flag.enc file by reversing the encryption algorithm:
1. Extract timestamp from first 4 bytes
2. Seed C-style random number generator with timestamp
3. For each encrypted byte, reverse: rotation -> XOR
"""
# Read the encrypted file
with open(filename, 'rb') as f:
data = f.read()
# Extract timestamp (first 4 bytes, little-endian unsigned int)
timestamp = struct.unpack('<I', data[:4])[0]
print(f"Extracted timestamp: {timestamp}")
print(f"Timestamp in hex: 0x{timestamp:08x}")
# Get the encrypted flag data (skip first 4 bytes)
encrypted_flag = data[4:]
print(f"Encrypted flag length: {len(encrypted_flag)} bytes")
print(f"Encrypted flag (hex): {encrypted_flag.hex()}")
# Set up C-style random number generator
# Python's random uses a different algorithm, so we use ctypes to call C's rand()
libc = ctypes.CDLL("libc.so.6")
libc.srand(timestamp)
decrypted = bytearray()
for i, encrypted_byte in enumerate(encrypted_flag):
# Get the same random numbers the encryptor used
rand1 = libc.rand() & 0xFFFFFFFF # Ensure 32-bit
rand2 = libc.rand() & 0xFFFFFFFF
print(f"Byte {i}: encrypted=0x{encrypted_byte:02x}, rand1={rand1}, rand2={rand2}")
# Reverse the encryption (in reverse order):
# Step 1: Reverse the rotation (rotate right instead of left)
rotation_amount = rand2 & 7 # Same as encryption: keep lower 3 bits
# Right rotation: opposite of left rotation
decrypted_byte = ((encrypted_byte >> rotation_amount) |
(encrypted_byte << (8 - rotation_amount))) & 0xFF
print(f" After reversing rotation (amount={rotation_amount}): 0x{decrypted_byte:02x}")
# Step 2: Reverse the XOR
xor_key = rand1 & 0xFF # Low byte of first random number
decrypted_byte ^= xor_key
print(f" After reversing XOR (key=0x{xor_key:02x}): 0x{decrypted_byte:02x} = '{chr(decrypted_byte) if 32 <= decrypted_byte <= 126 else '?'}'")
decrypted.append(decrypted_byte)
return decrypted
if __name__ == "__main__":
try:
decrypted_flag = decrypt_flag("flag.enc")
print(f"\n{'='*50}")
print(f"DECRYPTED FLAG: {decrypted_flag.decode('utf-8', errors='replace')}")
print(f"Flag in hex: {decrypted_flag.hex()}")
print(f"{'='*50}")
# Save decrypted flag to file
with open("decrypted_flag.txt", "w") as f:
f.write(decrypted_flag.decode('utf-8', errors='replace'))
print("Decrypted flag saved to 'decrypted_flag.txt'")
except Exception as e:
print(f"Error during decryption: {e}")
# Debug: Show raw hex values
with open("flag.enc", "rb") as f:
data = f.read()
print(f"Raw flag.enc hex: {data.hex()}")
Output:
1
2
3
4
5
==================================================
DECRYPTED FLAG: HTB{vRy_s*****}
Flag in hex: 48544f****
==================================================
Decrypted flag saved to 'decrypted_flag.txt'
This post is licensed under CC BY 4.0 by the author.