SourcePP
Several modern C++20 libraries for sanely parsing Valve's formats.
Loading...
Searching...
No Matches
GCF.cpp
Go to the documentation of this file.
1#include <vpkpp/format/GCF.h>
2
3#include <algorithm>
4#include <filesystem>
5
6#include <cryptopp/aes.h>
7#include <cryptopp/modes.h>
8#include <cryptopp/filters.h>
9#include <FileStream.h>
10#include <miniz.h>
13
14using namespace sourcepp;
15using namespace vpkpp;
16
17std::unique_ptr<PackFile> GCF::open(const std::string& path, const EntryCallback& callback, const OpenPropertyRequest& requestProperty) {
18 // TODO: Add v5 and perhaps v4 support
19
20 if (!std::filesystem::exists(path)) {
21 // File does not exist
22 return nullptr;
23 }
24
25 auto* gcf = new GCF{path};
26 auto packFile = std::unique_ptr<PackFile>(gcf);
27
28 FileStream reader(gcf->fullFilePath);
29 reader.seek_in(0);
30
31 // read the main header here (not the block header)
32 reader.read(gcf->header);
33 if (gcf->header.dummy1 != 1 && gcf->header.dummy2 != 1 && gcf->header.gcfversion != 6) {
40 return nullptr;
41 }
42
43 if (gcf->header.filesize != std::filesystem::file_size(gcf->fullFilePath)) {
44 // again, this should never occur with a valid gcf file
45 return nullptr;
46 }
47
48 reader.read(gcf->blockheader);
49 if (gcf->blockheader.count != gcf->header.blockcount) {
50 return nullptr;
51 }
52
53 // if you're having a headache reading the if statement below heres a quick explaination of what it actually does
54 // it just adds all blockheader entries together and compares it against the checksum
55 // if it's not the same then we bail out with a nullptr
56 if (gcf->blockheader.count +
57 gcf->blockheader.used +
58 gcf->blockheader.dummy1 +
59 gcf->blockheader.dummy2 +
60 gcf->blockheader.dummy3 +
61 gcf->blockheader.dummy4 +
62 gcf->blockheader.dummy5 != gcf->blockheader.checksum) {
63 return nullptr;
64 }
65
66 // block headers!!!!!!
67 bool requestKey = false;
68 for (int i = 0; i < gcf->header.blockcount; i++) {
69 Block& block = gcf->blockdata.emplace_back();
70 reader.read(block);
71 if (block.isEncrypted()) {
72 requestKey = true;
73 }
74 }
75 if (requestKey) {
76 const auto key = requestProperty(gcf, OpenProperty::DECRYPTION_KEY);
77 if (key.size() != gcf->decryption_key.size()) {
78 return nullptr;
79 }
80 std::ranges::copy(key, gcf->decryption_key.begin());
81 }
82
83 // Fragmentation Map header
84 // not worth keeping around after verifying stuff so no struct def
85 // if you want one this is how one should look like
92 // actually if we want to implement writing later this might be a better idea
93 // if anyone wants to touch this piece of shit format again anyways
94
95 auto blkcount = reader.read<uint32_t>();
96 auto d1 = reader.read<uint32_t>();
97 auto d2 = reader.read<uint32_t>();
98 auto checksum = reader.read<uint32_t>();
99
100 if (blkcount + d1 + d2 != checksum || blkcount != gcf->blockheader.count) {
101 return nullptr;
102 }
103
104 // Fragmentation Map (list of dwords)
105
106 for (int i = 0; i < blkcount; i++) {
107 gcf->fragmap.push_back(reader.read<uint32_t>());
108 }
109
110 // Directory stuff starts here
111
112 //Reading the header
113 uint64_t temp = reader.tell_in();
114 reader.read(gcf->dirheader);
115
116 uint64_t diroffset = reader.tell_in() + (gcf->dirheader.itemcount * 28);
117
118 std::vector<DirectoryEntry2> direntries{};
119
120 for (int i = 0; i < gcf->dirheader.itemcount; i++) {
121 DirectoryEntry2& entry = direntries.emplace_back();
122 reader.read(entry.entry_real);
123 auto currentoffset = reader.tell_in();
124 reader.seek_in_u(diroffset + entry.entry_real.nameoffset);
125 reader.read(entry.filename);
126 if (entry.entry_real.dirtype != 0) { // if not directory
127 std::string dirname;
128 DirectoryEntry2 current_filename_entry = entry;
129 while (current_filename_entry.entry_real.parentindex != 0xffffffff) {
130 current_filename_entry = direntries[current_filename_entry.entry_real.parentindex];
131 dirname.insert(0, "/");
132 dirname.insert(0, current_filename_entry.filename);
133 }
134
135 auto gcfEntryPath = gcf->cleanEntryPath(dirname);
136 if (!gcfEntryPath.empty()) {
137 gcfEntryPath += '/';
138 }
139 gcfEntryPath += entry.filename;
140
141 Entry gcfEntry = createNewEntry();
142 gcfEntry.length = entry.entry_real.itemsize;
143 gcfEntry.crc32 = entry.entry_real.fileid; // INDEX INTO THE CHECKSUM MAP VECTOR NOT CRC32!!!
144 gcfEntry.offset = i; // THIS IS THE STRUCT INDEX NOT SOME OFFSET!!!
145 //printf("%s\n", gcfEntry.path.c_str());
146 gcf->entries.emplace(gcfEntryPath, gcfEntry);
147
148 if (callback) {
149 callback(gcfEntryPath, gcfEntry);
150 }
151 }
152 reader.seek_in_u(currentoffset);
153 }
154
155 // Directory Map
156
157 // Directory Map header
158 reader.seek_in_u(temp + gcf->dirheader.dirsize);
159
160 //auto dmap = reader.read<DirectoryMapHeader>();
161 reader.skip_in<DirectoryMapHeader>();
162
163 // Directory Map entries
164 for (int i = 0; i < gcf->dirheader.itemcount; i++) {
165 DirectoryMapEntry& entry = gcf->dirmap_entries.emplace_back();
166 reader.read(entry);
167 }
168
169 // Checksum header
170 //auto dummy0 = reader.read<uint32_t>();
171 reader.skip_in<uint32_t>();
172 auto checksumsize = reader.read<uint32_t>();
173 std::size_t checksums_start = reader.tell_in();
174
175 //printf("checksums start: %llu\n", checksums_start);
176 //printf("%lu %lu %lu %lu\n", gcf->header.blockcount, gcf->blockheader.used, gcf->header.appid, gcf->header.appversion);
177 // map header
178
179 auto chksummapheader = reader.read<ChecksumMapHeader>();
180 if (chksummapheader.dummy1 != 0x14893721 || chksummapheader.dummy2 != 0x1) {
181 return nullptr;
182 }
183
184 //printf("%lu %lu\n", chksummapheader.checksum_count, chksummapheader.item_count);
185
186 for (int i = 0; i < chksummapheader.item_count; i++) {
187 auto& cur_entry = gcf->chksum_map.emplace_back();
188 reader.read(cur_entry);
189 }
190
191 for (int i = 0; i < chksummapheader.checksum_count; i++) {
192 auto& currentChecksum = gcf->checksums.emplace_back();
193 reader.read(currentChecksum);
194 }
195 //printf("current pos: %llu, block header: %llu should be: %llu", reader.tellInput(), reader.tellInput() + 0x80, checksums_start + checksumsize);
196 // TODO: check the checksum RSA signature... later.. if ever...
197
198 reader.seek_in_u(checksums_start + checksumsize);
199
200 reader.read(gcf->datablockheader);
201 return packFile;
202}
203
204std::vector<std::string> GCF::verifyEntryChecksums() const {
205 std::vector<std::string> bad;
206 this->runForAllEntries([this, &bad](const std::string& path, const Entry& entry) {
207 auto bytes = this->readEntry(path);
208 if (!bytes || bytes->empty()) {
209 return;
210 }
211 std::size_t tocheck = bytes->size();
212 uint32_t idx = entry.crc32;
213 uint32_t count = this->chksum_map[idx].count;
214 uint32_t checksumstart = this->chksum_map[idx].firstindex;
215 for (int i = 0; i < count; i++) {
216 uint32_t csum = this->checksums[checksumstart + i];
217 std::size_t toread = std::min(static_cast<std::size_t>(0x8000), tocheck);
218 const auto* data = bytes->data() + (i * 0x8000);
219 uint32_t checksum = crypto::computeCRC32({data, toread}) ^ crypto::computeAdler32({data, toread});
220 if (checksum != csum) {
221 bad.push_back(path);
222 }
223 tocheck -= toread;
224 }
225 });
226 return bad;
227}
228
229std::optional<std::vector<std::byte>> GCF::readEntry(const std::string& path_) const {
230 auto path = this->cleanEntryPath(path_);
231 auto entry = this->findEntry(path);
232 if (!entry) {
233 return std::nullopt;
234 }
235 if (entry->unbaked) {
236 return readUnbakedEntry(*entry);
237 }
238
239 std::vector<std::byte> filedata;
240 if (entry->length == 0) {
241 // don't bother
242 return filedata;
243 }
244
245 uint32_t dir_index = entry->offset;
246 //printf(" extracting file: %s\n", entry.path.c_str());
247
248 std::vector<Block> toread;
249 for (const auto& v : this->blockdata) {
250 if (v.dir_index == dir_index && (v.flags & 0x8000)) { // need to check for 0x8000 here because valve dumps uninitialized memory
251 toread.push_back(v);
252 }
253 }
254
255 if (toread.empty()) {
256 //printf("could not find any directory index for %lu", entry.vpk_offset);
257 return std::nullopt;
258 }
259 std::sort(toread.begin(), toread.end(), [](const Block& lhs, const Block& rhs) {
260 return lhs.file_data_offset < rhs.file_data_offset;
261 });
262
263 FileStream stream{this->fullFilePath};
264 if (!stream) {
265 return std::nullopt;
266 }
267
268 uint64_t remaining = entry->length;
269 bool needs_decrypt = false;
271 for (auto& block : toread) {
272 uint32_t currindex = block.first_data_block_index;
273 while (currindex <= this->blockheader.count) {
274 uint64_t curfilepos = static_cast<uint64_t>(this->datablockheader.firstblockoffset) + (static_cast<std::uint64_t>(0x2000) * static_cast<std::uint64_t>(currindex));
275 stream.seek_in_u(curfilepos);
276 //printf("off %lli block %lu toread %lli should be %llu\n", stream.tellInput(), currindex, remaining, curfilepos);
277 uint32_t toreadAmt = std::min(remaining, static_cast<uint64_t>(0x2000));
278 auto streamvec = stream.read_bytes(toreadAmt);
279 filedata.insert(filedata.end(), streamvec.begin(), streamvec.end());
280 remaining -= toreadAmt;
281 currindex = this->fragmap[currindex];
282 //printf("curridx now: %lu\n", currindex);
283 }
284 needs_decrypt = block.isEncrypted();
285 filemode = block.getCompressionType();
286 }
287 if (needs_decrypt) {
288 CryptoPP::byte iv[16] = {};
289 switch (filemode) {
290 using enum Block::CompressionType;
291 case UNCOMPRESSED:
292 case COMPRESSED:
293 break;
294 case ENCRYPTED: {
295 auto real_size = filedata.size();
296 if (filedata.size() % 10 != 0) {
297 filedata.resize(filedata.size() + (0x10 - (filedata.size() % 10)), {});
298 }
299 auto remaining_encrypted = filedata.size();
300 auto offset = 0;
301 while (remaining_encrypted) {
302 CryptoPP::CFB_Mode<CryptoPP::AES>::Decryption d;
303 d.SetKeyWithIV(reinterpret_cast<const CryptoPP::byte*>(this->decryption_key.data()), 16, iv);
304 auto current_batch = std::min(remaining_encrypted, static_cast<size_t>(0x8000));
305 d.ProcessData(reinterpret_cast<CryptoPP::byte*>(filedata.data() + offset), reinterpret_cast<CryptoPP::byte*>(filedata.data() + offset), current_batch);
306 remaining_encrypted -= current_batch;
307 offset += static_cast<int>(current_batch);
308 }
309 filedata.resize(real_size);
310 break;
311 }
312 case COMPRESSED_AND_ENCRYPTED: {
313 std::vector<std::byte> processed_data;
314 BufferStream s(filedata.data(), filedata.size());
315 while (s.size() != s.tell()) {
316 static constexpr auto allocate_block = [](std::vector<std::byte>& vec, size_t x) {
317 const size_t old = vec.size();
318 vec.resize(old + x);
319 return vec.data() + old;
320 };
321
322 auto encrypted_size = s.read<uint32_t>();
323 mz_ulong decompressed_size = s.read<uint32_t>();
324
325 auto buffer = s.read_bytes(encrypted_size);
326 auto real_size = buffer.size();
327 if (buffer.size() % 10 != 0) {
328 buffer.resize(buffer.size() + (0x10 - (buffer.size() % 10)), {});
329 }
330 CryptoPP::CFB_Mode<CryptoPP::AES>::Decryption d;
331 d.SetKeyWithIV(reinterpret_cast<const CryptoPP::byte*>(this->decryption_key.data()), 16, iv);
332 d.ProcessData(reinterpret_cast<CryptoPP::byte*>(buffer.data()), reinterpret_cast<CryptoPP::byte*>(buffer.data()), buffer.size());
333 buffer.resize(real_size);
334
335 auto start = allocate_block(processed_data, decompressed_size);
336 if (mz_uncompress(reinterpret_cast<uint8_t*>(start), &decompressed_size, reinterpret_cast<uint8_t*>(buffer.data()), buffer.size()) != MZ_OK) {
337 return std::nullopt;
338 }
339
340 //auto remaining = s.size() - s.tell();
341 if (entry->length == processed_data.size()) {
342 break;
343 }
344 s.seek_u(s.tell() + (0x8000 - s.tell() % 0x8000));
345 }
346 filedata = processed_data;
347 break;
348 }
349 }
350 }
351 return filedata;
352}
353
355 using enum Attribute;
356 return LENGTH;
357}
358
359GCF::operator std::string() const {
360 return PackFileReadOnly::operator std::string() +
361 " | Version v" + std::to_string(this->header.gcfversion) +
362 " | AppID " + std::to_string(this->header.appid) +
363 " | App Version v" + std::to_string(this->header.appversion);
364}
365
366uint32_t GCF::getVersion() const {
367 return this->header.gcfversion;
368}
369
370uint32_t GCF::getAppID() const {
371 return this->header.appid;
372}
373
374uint32_t GCF::getAppVersion() const {
375 return this->header.appversion;
376}
This class represents the metadata that a file has inside a PackFile.
Definition: Entry.h:14
uint64_t offset
Offset, format-specific meaning - 0 if unused, or if the offset genuinely is 0.
Definition: Entry.h:33
uint32_t crc32
CRC32 checksum - 0 if unused.
Definition: Entry.h:40
uint64_t length
Length in bytes (in formats with compression, this is the uncompressed length)
Definition: Entry.h:26
Definition: GCF.h:9
static std::unique_ptr< PackFile > open(const std::string &path, const EntryCallback &callback=nullptr, const OpenPropertyRequest &requestProperty=nullptr)
Definition: GCF.cpp:17
Header header
Definition: GCF.h:172
std::vector< Block > blockdata
Definition: GCF.h:174
BlockHeader blockheader
Definition: GCF.h:173
std::vector< ChecksumMapEntry > chksum_map
Definition: GCF.h:180
std::array< std::byte, 16 > decryption_key
Definition: GCF.h:182
uint32_t getVersion() const
Definition: GCF.cpp:366
std::vector< std::string > verifyEntryChecksums() const override
Verify the checksums of each file, if a file fails the check its path will be added to the vector If ...
Definition: GCF.cpp:204
uint32_t getAppVersion() const
Definition: GCF.cpp:374
DataBlockHeader datablockheader
Definition: GCF.h:179
Attribute getSupportedEntryAttributes() const override
Returns a list of supported entry attributes Mostly for GUI programs that show entries and their meta...
Definition: GCF.cpp:354
std::vector< uint32_t > checksums
Definition: GCF.h:181
uint32_t getAppID() const
Definition: GCF.cpp:370
std::optional< std::vector< std::byte > > readEntry(const std::string &path_) const override
Try to read the entry's data to a bytebuffer.
Definition: GCF.cpp:229
std::vector< uint32_t > fragmap
Definition: GCF.h:175
EntryCallbackBase< void > EntryCallback
Definition: PackFile.h:37
std::optional< Entry > findEntry(const std::string &path_, bool includeUnbaked=true) const
Try to find an entry given the file path.
Definition: PackFile.cpp:166
std::string fullFilePath
Definition: PackFile.h:229
std::function< std::vector< std::byte >(PackFile *packFile, OpenProperty property)> OpenPropertyRequest
Definition: PackFile.h:32
void runForAllEntries(const EntryCallback &operation, bool includeUnbaked=true) const
Run a callback for each entry in the pack file.
Definition: PackFile.cpp:503
std::string cleanEntryPath(const std::string &path) const
Definition: PackFile.cpp:680
static Entry createNewEntry()
Definition: PackFile.cpp:689
static std::optional< std::vector< std::byte > > readUnbakedEntry(const Entry &entry)
Definition: PackFile.cpp:693
uint32_t computeAdler32(std::span< const std::byte > buffer)
Definition: Adler32.cpp:39
uint32_t computeCRC32(std::span< const std::byte > buffer)
Definition: CRC32.cpp:7
Definition: LZMA.h:11
Definition: Attribute.h:5
Attribute
Definition: Attribute.h:7
uint32_t count
Definition: GCF.h:29
bool isEncrypted() const
Definition: GCF.h:70
uint32_t firstblockoffset
Definition: GCF.h:120
std::string filename
Definition: GCF.h:104
DirectoryEntry entry_real
Definition: GCF.h:103
uint32_t nameoffset
Definition: GCF.h:93
uint32_t parentindex
Definition: GCF.h:97
uint32_t appversion
Definition: GCF.h:17
uint32_t appid
Definition: GCF.h:16
uint32_t gcfversion
Definition: GCF.h:15