From 43e539561a4ea95274468b78735555d6e89fdfd9 Mon Sep 17 00:00:00 2001 From: jakob Date: Thu, 12 Jan 2017 17:09:17 -0500 Subject: Finished implementation of file decryption. There are still many memory corruption bugs and a segfault is not uncommon. --- .gitignore | 2 -- GNUmakefile | 2 +- TODO.md | 1 + src/cli.c | 4 +-- src/crypto.c | 76 +++++++++++++++++++++++++++++++++++++++++++++++ src/crypto.h | 39 +++++++++++++++++++++++++ src/decompress.c | 50 ++++++++++++++++--------------- src/decompress.h | 8 +++-- src/extract.c | 38 ++++++++++-------------- src/extract.h | 1 - src/file.c | 24 +++++++-------- src/file.h | 8 ++--- src/main.c | 5 ++-- src/write.c | 89 ++++++++++++++++++++++++++++++-------------------------- src/write.h | 14 ++++++++- 15 files changed, 243 insertions(+), 118 deletions(-) create mode 100644 src/crypto.c create mode 100644 src/crypto.h diff --git a/.gitignore b/.gitignore index a0f403f..cd42ee3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,2 @@ bin/ obj/ -dump.bin -README.md.new diff --git a/GNUmakefile b/GNUmakefile index 006a328..0238712 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -5,7 +5,7 @@ SHELL = /bin/sh CC := gcc LD := gcc -CFLAGS := -Wall -Wextra -Os -funroll-all-loops -std=c99 -pedantic +CFLAGS := -Wall -Wextra -Os -funroll-all-loops -std=gnu99 -pedantic LDFLAGS := -lz STRIPARGS := -s -p diff --git a/TODO.md b/TODO.md index 9612929..4889310 100644 --- a/TODO.md +++ b/TODO.md @@ -1,3 +1,4 @@ TODO ==== * Write a 100% compatible repacker to aid in translation efforts. +* Functionality for just listing the archive's contents. diff --git a/src/cli.c b/src/cli.c index 2713fed..6144c4e 100644 --- a/src/cli.c +++ b/src/cli.c @@ -44,7 +44,7 @@ "configuration" structure containing everything parsed from argv. */ struct configuration parse_args(int argc, char *argv[]) { if (argc < 2) { - fprintf(stderr, "Usage: %s (ARCHIVE PATH) [OPTIONS]\n", argv[0]); + fprintf(stderr, "Usage: %s [OPTIONS] (ARCHIVE PATH)\n", argv[0]); exit(1); } @@ -102,7 +102,7 @@ struct configuration parse_args(int argc, char *argv[]) { first. argv[count] is the first positional argument encountered. */ if (parsed.archive_path == NULL) { if (argv[count] == NULL) { - fprintf(stderr, "No archive path given.\n"); + fprintf(stderr, "No archive path provided.\n"); exit(EXIT_FAILURE); } parsed.archive_path = argv[count]; diff --git a/src/crypto.c b/src/crypto.c new file mode 100644 index 0000000..2d527c7 --- /dev/null +++ b/src/crypto.c @@ -0,0 +1,76 @@ +/* This file is part of Nekopack. + + Copyright (C) 2017 Jakob Tsar-Fox, All Rights Reserved. + + Nekopack is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at + your option) any later version. + + Nekopack is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with Nekopack. If not, see . */ + +#include "cli.h" +#include "crypto.h" + + +/* Returns the encryption keys for a given game value. */ +key get_encryption_key(game current_game) { + key encryption_key; + switch (current_game) { + case NEKOPARA_VOLUME_0: + encryption_key.master_key = 0x1548e29c; + encryption_key.initial_fallback_key = 0x9c; + encryption_key.primary_fallback_key = 0xd7; + encryption_key.uses_initial_key = 1; + break; + case NEKOPARA_VOLUME_0_STEAM: + encryption_key.master_key = 0x44528b87; + encryption_key.initial_fallback_key = 0x87; + encryption_key.primary_fallback_key = 0x23; + encryption_key.uses_initial_key = 1; + break; + case NEKOPARA_VOLUME_1: + encryption_key.master_key = 0x1548e29c; + encryption_key.initial_fallback_key = 0x00; + encryption_key.primary_fallback_key = 0xd7; + encryption_key.uses_initial_key = 0; + break; + case NEKOPARA_VOLUME_1_STEAM: + encryption_key.master_key = 0x44528b87; + encryption_key.initial_fallback_key = 0x00; + encryption_key.primary_fallback_key = 0x23; + encryption_key.uses_initial_key = 0; + break; + default: + encryption_key.master_key = 0x00000000; + encryption_key.initial_fallback_key = 0x00; + encryption_key.primary_fallback_key = 0x00; + encryption_key.uses_initial_key = 0; + } + return encryption_key; +} + + +/* Decrypts the contents of a buffer according to a file key. */ +void decrypt_buffer(Bytef *encrypted_buffer, uint64_t buffer_length, + key encryption_key, uint32_t file_key) { + uint32_t xor_key = file_key ^ encryption_key.master_key; + uint8_t initial_key = xor_key & 0xff; + uint8_t primary_key = (xor_key >> 24 ^ xor_key >> 16 ^ \ + xor_key >> 8 ^ xor_key) & 0xff; + if (xor_key == 1 && initial_key == 0) + initial_key = encryption_key.initial_fallback_key; + else if (primary_key == 0) + primary_key = encryption_key.primary_fallback_key; + if (encryption_key.uses_initial_key) + encrypted_buffer[0] ^= initial_key; + for (uint64_t i = 0; i < buffer_length; i++) { + encrypted_buffer[i] ^= primary_key; + } +} diff --git a/src/crypto.h b/src/crypto.h new file mode 100644 index 0000000..e82b934 --- /dev/null +++ b/src/crypto.h @@ -0,0 +1,39 @@ +/* This file is part of Nekopack. + + Copyright (C) 2017 Jakob Tsar-Fox, All Rights Reserved. + + Nekopack is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at + your option) any later version. + + Nekopack is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with Nekopack. If not, see . */ + +#pragma once + +#include + +#include + +#include "cli.h" + +/* Struct for a game's encryption keys. */ +typedef struct { + int uses_initial_key; /* Set if first byte uses a different key. */ + uint32_t master_key; /* Master key used to derive a key. */ + uint8_t initial_fallback_key; /* Fallback for the initial key. */ + uint8_t primary_fallback_key; /* Fallback for the primary key. */ +} key; + +/* Returns the encryption keys for a given game value. */ +key get_encryption_key(game current_game); + +/* Decrypts the contents of a buffer according to a file key. */ +void decrypt_buffer(Bytef *encrypted_buffer, uint64_t buffer_length, + key encryption_key, uint32_t file_key); diff --git a/src/decompress.c b/src/decompress.c index 0301cc8..12f6848 100644 --- a/src/decompress.c +++ b/src/decompress.c @@ -63,39 +63,41 @@ Bytef *inflate_chunk(Bytef *chunk, uint64_t chunk_size, } while (data_stream.avail_out == 0); } while (status_code != Z_STREAM_END); - inflateEnd(&data_stream); + // inflateEnd(&data_stream); return decompressed_data; } -/* Wrapper for inflate_chunk which operates on FILE pointers. The file - pointer's contents are inflated into a memory stream and returned. */ -memory_stream decompress_file(FILE *archive, uint64_t sizes_offset) { - uint64_t compressed_size, decompressed_size; - fseek(archive, sizes_offset, SEEK_SET); - fread(&compressed_size, sizeof(uint64_t), 1, archive); - fread(&decompressed_size, sizeof(uint64_t), 1, archive); - - /* Decompression is done in memory because it's $CURRENT_YEAR. */ - Bytef *decompressed_data, *compressed_data = malloc(compressed_size); - - /* This is a pretty shitty way of handling it, though. */ - if (compressed_data == NULL) { +/* Decompresses a memory_stream data structure into a new one. */ +memory_stream decompress_stream(memory_stream compressed_data, uint64_t size) { + Bytef *decompressed_data = malloc(size); + /* This is a pretty shitty way of handling it. */ + if (decompressed_data == NULL) { fprintf(stderr, "Insufficient memory to decompress archive.\n"); - free(compressed_data); - fclose(archive); + free(compressed_data.start); + free(decompressed_data); exit(EXIT_FAILURE); } - - fread(compressed_data, compressed_size, 1, archive); - decompressed_data = inflate_chunk(compressed_data, compressed_size, - decompressed_size); - /* The compressed data is irrelevant at this point. */ - free(compressed_data); + decompressed_data = inflate_chunk(compressed_data.start, + compressed_data.stream_length, + size); if (decompressed_data == NULL) exit(EXIT_FAILURE); + return (memory_stream) {size, decompressed_data, decompressed_data}; +} + - return (memory_stream) {decompressed_size, decompressed_data, - decompressed_data}; +/* Reads a FILE pointer into a memory_stream data structure. */ +memory_stream read_to_stream(FILE *archive, uint64_t buffer_size) { + Bytef *buffer = malloc(buffer_size); + /* This is a pretty shitty way of handling it. */ + if (buffer == NULL) { + fprintf(stderr, "Insufficient memory to load archive.\n"); + free(buffer); + fclose(archive); + exit(EXIT_FAILURE); + } + fread(buffer, buffer_size, 1, archive); + return (memory_stream) {buffer_size, buffer, buffer}; } diff --git a/src/decompress.h b/src/decompress.h index a625f5e..dcb985b 100644 --- a/src/decompress.h +++ b/src/decompress.h @@ -21,9 +21,11 @@ #include "extract.h" -/* Wrapper for inflate_chunk which operates on FILE pointers. The file - pointer's contents are inflated into a memory stream and returned. */ -memory_stream decompress_file(FILE *archive, uint64_t sizes_offset); +/* Decompresses a memory_stream data structure into a new one. */ +memory_stream decompress_stream(memory_stream compressed_data, uint64_t size); + +/* Reads a FILE pointer into a memory_stream data structure. */ +memory_stream read_to_stream(FILE *archive, uint64_t buffer_size); /* Returns a pointer to a buffer containing the inflated contents of a given memory chunk. */ diff --git a/src/extract.c b/src/extract.c index f452eaf..1dad149 100644 --- a/src/extract.c +++ b/src/extract.c @@ -44,33 +44,28 @@ void read_stream(void *destination, Bytef **source, size_t size); parsing, decrypting and writing the table entries. */ void extract(FILE *archive, uint64_t table_offset) { /* eliF and File entries are stored in a linked list as they're - seen, because the order of entries in XP3 archives is not + seen because the order of entries in XP3 archives is not guaranteed to be chronological. calloc is used to prevent the "next" pointer from being garbage and causing a segfault. */ - elif_node *elif_root = calloc(sizeof(elif_node), 1); - elif_node *elif_new; - file_node *file_root = calloc(sizeof(file_node), 1); - file_node *file_new; + elif_node *elif_new, *elif_root = calloc(sizeof(elif_node), 1); + file_node *file_new, *file_root = calloc(sizeof(file_node), 1); uint8_t compressed; + uint64_t compressed_size, decompressed_size; fseek(archive, table_offset, SEEK_SET); fread(&compressed, sizeof(uint8_t), 1, archive); + fread(&compressed_size, sizeof(uint64_t), 1, archive); + fread(&decompressed_size, sizeof(uint64_t), 1, archive); int stream_ended = 0; memory_stream data_stream; + memory_stream compressed_data = read_to_stream(archive, compressed_size); if (compressed) { - data_stream = decompress_file(archive, ftell(archive)); + data_stream = decompress_stream(compressed_data, decompressed_size); + free(compressed_data.start); } else { - uint64_t decompressed_size; - fread(&decompressed_size, sizeof(uint64_t), 1, archive); - /* The second size is irrelevant and therefore ignored. */ - fseek(archive, sizeof(uint64_t), SEEK_CUR); - data_stream.stream_length = decompressed_size; - data_stream.data = malloc(decompressed_size); - data_stream.start = data_stream.data; - fread(data_stream.data, decompressed_size, 1, archive); - fclose(archive); - } + data_stream = compressed_data; + } uint32_t entry_magic; uint64_t entry_size; @@ -102,9 +97,7 @@ void extract(FILE *archive, uint64_t table_offset) { } } while (!stream_ended); - write_files(file_root, elif_root, data_stream.start); - test_elif_linked_list(elif_root); - test_file_linked_list(file_root); + write_files(file_root, elif_root, archive); free_elif_nodes(elif_root); free_file_nodes(file_root); free(data_stream.start); @@ -129,8 +122,8 @@ elif_node *read_elif_entry(memory_stream *data_stream) { char *file_name; if (name_size < 0x100) { - /* Strings are terminated by null bytes, but - they aren't counted in the name size. */ + /* Strings are terminated by null bytes, + which aren't counted in the name size. */ char *input_buffer = malloc(name_size * 2 + 2); file_name = malloc(name_size + 1); read_stream(input_buffer, &data_stream->data, name_size * 2 + 2); @@ -145,8 +138,7 @@ elif_node *read_elif_entry(memory_stream *data_stream) { free(input_buffer); } else { - /* It's pretty safe to assume anything - larger is the copyright notice. */ + /* It's safe to assume too large is the copyright notice. */ data_stream->data += name_size * 2 + 2; file_name = strdup("COPYING.txt"); } diff --git a/src/extract.h b/src/extract.h index 0247ea9..9aea965 100644 --- a/src/extract.h +++ b/src/extract.h @@ -37,4 +37,3 @@ void extract(FILE *archive, uint64_t table_offset); /* Wrapper for memcpy which increments the source operand by the amount of bytes read to simulate a file stream. */ void read_stream(void *destination, Bytef **source, size_t size); - diff --git a/src/file.c b/src/file.c index aa7e00d..0c5751a 100644 --- a/src/file.c +++ b/src/file.c @@ -50,7 +50,8 @@ file_node *read_file_entry(memory_stream *data_stream, Bytef *section_end) { break; case SEGM_MAGIC: printf("[SEGM found at 0x%lx]\n", data_stream->data - data_stream->start - 12); - read_segm_chunk(data_stream, parsed, entry_size); + /* Segments are 28 bytes each. */ + read_segm_chunk(data_stream, parsed, entry_size / 28); break; case INFO_MAGIC: printf("[INFO found at 0x%lx]\n", data_stream->data - data_stream->start - 12); @@ -69,7 +70,7 @@ file_node *read_file_entry(memory_stream *data_stream, Bytef *section_end) { } -/* Document */ +/* Reads the contents of an info chunk int a file_node. */ void read_info_chunk(memory_stream *data_stream, file_node *parsed) { uint32_t encrypted; uint64_t decompressed_size, compressed_size; @@ -93,20 +94,18 @@ void read_info_chunk(memory_stream *data_stream, file_node *parsed) { } printf("\n"); + parsed->encrypted = encrypted; + data_stream->data += 2; } -/* Document */ +/* Reads the contents of a segm chunk into a file_node. */ void read_segm_chunk(memory_stream *data_stream, file_node *parsed, uint64_t segment_count) { - /* The segment_count is in bytes, with each - segment being 28 bytes in length. */ - segment_count /= 28; - /* Segments are stored in an array of segment pointers. */ segment **segments = malloc(sizeof(segment *) * segment_count); - for (int i = 0; i < segment_count; i++) { + for (uint64_t i = 0; i < segment_count; i++) { segments[i] = malloc(sizeof(segment)); read_stream(&segments[i]->compressed, &data_stream->data, @@ -120,11 +119,12 @@ void read_segm_chunk(memory_stream *data_stream, file_node *parsed, read_stream(&segments[i]->compressed_size, &data_stream->data, sizeof(uint64_t)); + parsed->file_size += segments[i]->decompressed_size; } printf("\nSEGMENT SECTION\n---------------\n\n"); printf("SEGMENT_COUNT: %" PRIu64 "\n", segment_count); - for (int i = 0; i < segment_count; i++) { - printf("\SEGMENT %d\n----------\n", i); + for (uint64_t i = 0; i < segment_count; i++) { + printf("\nSEGMENT %" PRIu64 "\n----------\n", i); printf("%s\n", segments[i]->compressed ? "COMPRESSED" : "DECOMPRESSED"); printf("FILE_OFFSET: %" PRIu64 "\n", segments[i]->offset); printf("COMPRESSED_SIZE: %" PRIu64 "\n", segments[i]->compressed_size); @@ -135,7 +135,7 @@ void read_segm_chunk(memory_stream *data_stream, file_node *parsed, } -/* Document */ +/* Reads the contents of an adlr chunk into a file_node. */ void read_adlr_chunk(memory_stream *data_stream, file_node *parsed) { uint32_t key; read_stream(&key, &data_stream->data, sizeof(uint32_t)); @@ -143,7 +143,7 @@ void read_adlr_chunk(memory_stream *data_stream, file_node *parsed) { } -/* Document */ +/* Reads the contents of a time chunk into a file_node. */ void read_time_chunk(memory_stream *data_stream, file_node *parsed) { uint64_t timestamp; read_stream(×tamp, &data_stream->data, sizeof(uint64_t)); diff --git a/src/file.h b/src/file.h index dc96154..046cd48 100644 --- a/src/file.h +++ b/src/file.h @@ -29,14 +29,14 @@ typedef struct { /* Node in a linked list of file entries to write to disk. */ typedef struct file_node { - int compressed; /* Whether or not the archive is compressed. */ + int encrypted; /* Whether or not the entry is encrypted. */ + int compressed; /* Whether or not the entry is compressed. */ uint32_t key; /* Key associated with matching eliF entry. */ - uint64_t compressed_size; /* Size of compressed chunk. */ - uint64_t decompressed_size; /* Size of decompressed data. */ + uint64_t file_size; /* Total size of decompressed file. */ uint64_t segment_count; /* Number of segments in File entry. */ segment **segments; /* Data segments associated with the entry. */ struct file_node *next; /* Pointer to the next node. */ } file_node; /* Creates a file node by parsing a file entry. */ -file_node *read_file_node(memory_stream *data_stream, Bytef *section_end); +file_node *read_file_entry(memory_stream *data_stream, Bytef *section_end); diff --git a/src/main.c b/src/main.c index 9effc9e..662acee 100644 --- a/src/main.c +++ b/src/main.c @@ -97,10 +97,9 @@ uint64_t get_table_offset(FILE *archive, uint8_t archive_version) { fprintf(stderr, "Minor version not implemented.\n"); fclose(archive); } - /* The read table_offset is actually an offset to the real - table offset. XP3 Version 2 is a little strange. */ + /* The read table_offset is an offset to the real table offset. */ fseek(archive, table_offset, SEEK_SET); - /* Flags and size of table are ignored in the parsing process. */ + /* Table flags and size are ignored in the parsing process. */ fseek(archive, sizeof(uint8_t) + sizeof(uint64_t), SEEK_CUR); fread(&table_offset, sizeof(uint64_t), 1, archive); return table_offset; diff --git a/src/write.c b/src/write.c index 1bd83da..f54d2f7 100644 --- a/src/write.c +++ b/src/write.c @@ -15,17 +15,19 @@ You should have received a copy of the GNU General Public License along with Nekopack. If not, see . */ -#include -#include #include #include #include +#include "crypto.h" #include "decompress.h" #include "extract.h" #include "file.h" #include "write.h" +/* Global instance of the command-line configuration structure. */ +extern struct configuration arguments; + /* Finds an eliF entry with the given key in a linked list and returns the associated filename, removing it from the linked list. */ @@ -51,33 +53,50 @@ char *pop_file_name(uint32_t key, elif_node *root) { /* Iterates through the linked list of file entries and writes every entry to disk, according to information specified by the node. */ -void write_files(file_node *file_root, elif_node *elif_root, Bytef *start) { - FILE *output; +void write_files(file_node *file_root, elif_node *elif_root, FILE *archive) { file_node *current; - Bytef *compressed_buffer, *decompressed_buffer; + key encryption_key = get_encryption_key(arguments.source); + Bytef *compressed_buffer, *decompressed_buffer, *out_buffer, *out_start; for (current = file_root->next; current != NULL; current = current->next) { char *file_name = pop_file_name(current->key, elif_root); - if (file_name == NULL) { - fprintf(stderr, "File found without matching eliF entry.\n"); + if (file_name == NULL) continue; + + /* out_start is kept so out_buffer can be incremented. */ + out_buffer = malloc(current->file_size); + out_start = out_buffer; + for (uint64_t i = 0; i < current->segment_count; i++) { + segment *chunk = current->segments[i]; + fseek(archive, chunk->offset, SEEK_SET); + compressed_buffer = malloc(chunk->compressed_size); + fread(compressed_buffer, chunk->compressed_size, 1, archive); + + if (chunk->compressed) { + decompressed_buffer = inflate_chunk(compressed_buffer, + chunk->compressed_size, + chunk->decompressed_size); + free(compressed_buffer); + } else { + decompressed_buffer = compressed_buffer; + } + memcpy(out_buffer, decompressed_buffer, chunk->decompressed_size); + out_buffer += chunk->decompressed_size; + free(decompressed_buffer); } - compressed_buffer = malloc(current->compressed_size); - for (int i = 0; i < current->segment_count; i++) - free(current->segments[i]); - free(current->segments); - /* memcpy(compressed_buffer, start + current->offset, */ - /* current->decompressed_size); */ - /* if (current->compressed) { */ - /* fprintf(stderr, "Not implemented :^)\n"); // don't leave this in lol */ - /* continue; */ - /* } else { */ - /* output = fopen(file_name, "wb+"); */ - /* fwrite(compressed_buffer, current->compressed_size, 1, output); */ - /* fclose(output); */ - /* } */ - - free(compressed_buffer); - break; + + if (arguments.source != NO_CRYPTO) { + decrypt_buffer(out_start, current->file_size, + encryption_key, current->key); + } + + FILE *output = fopen(file_name, "wb+"); + if (output == NULL) { + perror(file_name); + break; + } + fwrite(out_start, current->file_size, 1, output); + fclose(output); + free(out_start); } } @@ -112,8 +131,11 @@ void defer_file_node(file_node *new, file_node *root) { /* Iterates through the linked list and frees all entries. */ void free_elif_nodes(elif_node *base) { - if (base->next != NULL) + if (base->next != NULL) { free_elif_nodes(base->next); + } + if (base->file_name) + printf("%s\n", base->file_name); free(base->file_name); free(base); } @@ -125,20 +147,3 @@ void free_file_nodes(file_node *base) { free_file_nodes(base->next); free(base); } - - -void test_elif_linked_list(elif_node *root) { - for (elif_node *current = root; current != NULL; current = current->next) { - printf("ELIF NODE\n---------\n"); - printf("KEY: %" PRIx32 "\n", current->key); - printf("FILE_NAME: %s\n\n", current->file_name); - } -} - - -void test_file_linked_list(file_node *root) { - for (file_node *current = root; current != NULL; current = current->next) { - printf("FILE NODE\n---------\n"); - printf("KEY: %" PRIx32 "\n\n", current->key); - } -} diff --git a/src/write.h b/src/write.h index d37adda..7bd1fdc 100644 --- a/src/write.h +++ b/src/write.h @@ -25,4 +25,16 @@ typedef struct elif_node { /* Iterates through the linked list of file entries and writes every entry to disk, according to information specified by the node. */ -void write_files(file_node *file_root, elif_node *elif_root, Bytef *start); +void write_files(file_node *file_root, elif_node *elif_root, FILE *archive); + +/* Inserts an eliF entry at the end of a linked list. */ +void defer_elif_node(elif_node *new, elif_node *root); + +/* Inserts a File entry node at the end of a linked list. */ +void defer_file_node(file_node *new, file_node *root); + +/* Iterates through the linked list and frees all entries. */ +void free_elif_nodes(elif_node *base); + +/* Iterates through the linked list and frees all entries. */ +void free_file_nodes(file_node *base); -- cgit v1.3