Split oar library implementation up into modules
This commit is contained in:
parent
3b9e00b1cb
commit
1b6e80f3f1
458
source/oar.cpp
458
source/oar.cpp
@ -2,37 +2,96 @@ export module oar;
|
||||
|
||||
import coral;
|
||||
import coral.files;
|
||||
import coral.functional;
|
||||
import coral.io;
|
||||
import coral.stack;
|
||||
|
||||
using coral::closure;
|
||||
using coral::expected;
|
||||
using coral::file_reader;
|
||||
using coral::file_walker;
|
||||
using coral::fs;
|
||||
using coral::io_error;
|
||||
using coral::path;
|
||||
using coral::slice;
|
||||
using coral::u8;
|
||||
using coral::u64;
|
||||
using coral::usize;
|
||||
|
||||
/**
|
||||
export namespace oar {
|
||||
struct entry_path {
|
||||
enum class parse_error {
|
||||
|
||||
};
|
||||
|
||||
/**
|
||||
* Maximum path length.
|
||||
*/
|
||||
static usize const max = coral::u8_max;
|
||||
|
||||
/**
|
||||
* Common path component separator.
|
||||
*/
|
||||
static char const seperator = '/';
|
||||
|
||||
constexpr entry_path() {
|
||||
this->buffer[max] = max;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a weak reference to the path as a [slice].
|
||||
*/
|
||||
constexpr slice<char const> as_slice() const {
|
||||
return {this->buffer, this->filled()};
|
||||
}
|
||||
|
||||
/**
|
||||
* Compares the path to `that`, returning the difference between the two paths or `0` if they are identical.
|
||||
*/
|
||||
constexpr coral::size compare(entry_path const & that) const {
|
||||
return coral::compare(this->as_slice().as_bytes(), that.as_slice().as_bytes());
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the number of characters composing the path.
|
||||
*/
|
||||
constexpr usize filled() const {
|
||||
return max - this->buffer[max];
|
||||
}
|
||||
|
||||
static constexpr expected<entry_path, parse_error> parse(slice<char const> const & text) {
|
||||
// TODO: Implement.
|
||||
return entry_path{};
|
||||
}
|
||||
|
||||
private:
|
||||
char buffer[max + 1] {0};
|
||||
};
|
||||
|
||||
union entry_block {
|
||||
struct {
|
||||
entry_path path;
|
||||
|
||||
u64 data_offset;
|
||||
|
||||
u64 data_length;
|
||||
} layout;
|
||||
|
||||
u8 bytes[512];
|
||||
|
||||
static constexpr bool is_sizeof(usize value) {
|
||||
return value == sizeof(entry_block);
|
||||
}
|
||||
|
||||
expected<bool, coral::io_error> read(coral::reader & archive_reader) {
|
||||
return archive_reader.read(this->bytes).map<bool>(is_sizeof);
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Length of the full magic signature at the beginning of an Oar file.
|
||||
*/
|
||||
constexpr static usize signature_length {4};
|
||||
constexpr usize header_signature_length {4};
|
||||
|
||||
/**
|
||||
/**
|
||||
* Hardcoded signature magic value that this implementation of Oar expects when reading archives.
|
||||
*/
|
||||
constexpr static u8 signature_magic[signature_length] {'o', 'a', 'r', 1};
|
||||
constexpr u8 header_signature_magic[header_signature_length] {'o', 'a', 'r', 1};
|
||||
|
||||
/**
|
||||
* Oar file header format.
|
||||
*/
|
||||
union header {
|
||||
union header_block {
|
||||
struct {
|
||||
u8 signature[signature_length];
|
||||
u8 signature[header_signature_length];
|
||||
|
||||
coral::u32 entry_count;
|
||||
} layout;
|
||||
@ -40,370 +99,13 @@ union header {
|
||||
u8 bytes[512];
|
||||
|
||||
static constexpr bool is_sizeof(usize value) {
|
||||
return value == sizeof(header);
|
||||
return value == sizeof(header_block);
|
||||
}
|
||||
|
||||
expected<bool, io_error> read(coral::reader & archive_reader) {
|
||||
expected<bool, coral::io_error> read(coral::reader & archive_reader) {
|
||||
return archive_reader.read(this->bytes).map<bool>(is_sizeof).map<bool>([&](bool is_valid) -> bool {
|
||||
return is_valid && coral::equals(this->layout.signature, signature_magic);
|
||||
return is_valid && coral::equals(this->layout.signature, header_signature_magic);
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
static_assert(header::is_sizeof(512));
|
||||
|
||||
enum class entry_kind {
|
||||
file,
|
||||
directory,
|
||||
};
|
||||
|
||||
/**
|
||||
* Oar entry block format.
|
||||
*/
|
||||
union block {
|
||||
struct {
|
||||
path path;
|
||||
|
||||
u64 data_offset;
|
||||
|
||||
u64 data_length;
|
||||
|
||||
entry_kind kind;
|
||||
} layout;
|
||||
|
||||
u8 bytes[512];
|
||||
|
||||
static constexpr bool is_sizeof(usize value) {
|
||||
return value == sizeof(block);
|
||||
}
|
||||
|
||||
expected<bool, io_error> read(coral::reader & archive_reader) {
|
||||
return archive_reader.read(this->bytes).map<bool>(is_sizeof);
|
||||
}
|
||||
};
|
||||
|
||||
static_assert(block::is_sizeof(512));
|
||||
|
||||
/**
|
||||
* Archive entry access interface.
|
||||
*/
|
||||
struct entry final : public file_reader {
|
||||
/**
|
||||
* Results of a find operation performed on an [archive_file].
|
||||
*
|
||||
* [find_result::ok] means that the find operation was successful.
|
||||
*
|
||||
* [find_result::io_unavailable] signals a failure to communicate with the underlying [file_reader] for whatever
|
||||
* reason.
|
||||
*
|
||||
* [find_result::archive_invalid] signals that data was read but it does not match the format of an Oar archive.
|
||||
* This is typically because the underlying [file_reader] is not reading from an Oar archive file or the archive does
|
||||
* not match the supported version.
|
||||
*
|
||||
* [find_result::not_found] indicates that no entry in the archive could be found that matches the given query.
|
||||
*/
|
||||
enum class [[nodiscard]] find_result {
|
||||
ok,
|
||||
io_unavailable,
|
||||
archive_invalid,
|
||||
not_found,
|
||||
};
|
||||
|
||||
entry(file_reader * archive_reader) {
|
||||
this->archive_reader = archive_reader;
|
||||
}
|
||||
|
||||
/**
|
||||
* Performs a lookup for a file entry matching the path `file_path` in the archive, returning [find_result] to
|
||||
* indicate the result of the operation.
|
||||
*/
|
||||
find_result find(entry_kind kind, path const & entry_path) {
|
||||
this->data_offset = 0;
|
||||
this->data_length = 0;
|
||||
this->data_cursor = 0;
|
||||
|
||||
if (!this->archive_reader->seek(0).is_ok()) return find_result::io_unavailable;
|
||||
|
||||
header archive_header {};
|
||||
|
||||
if (!archive_header.read(*this->archive_reader).map<bool>(coral::equality_predicate(true)).is_ok())
|
||||
return find_result::archive_invalid;
|
||||
|
||||
// Read file table.
|
||||
u64 head {0};
|
||||
u64 tail {archive_header.layout.entry_count - 1};
|
||||
block archive_block {};
|
||||
|
||||
while (head <= tail) {
|
||||
u64 const midpoint {head + ((tail - head) / 2)};
|
||||
|
||||
if (!archive_block.read(*this->archive_reader).map<bool>(coral::equality_predicate(true)).is_ok())
|
||||
return find_result::archive_invalid;
|
||||
|
||||
if (archive_block.layout.kind == kind) return find_result::not_found;
|
||||
|
||||
coral::size const comparison {entry_path.compare(archive_block.layout.path)};
|
||||
|
||||
if (comparison == 0) {
|
||||
this->data_offset = archive_block.layout.data_offset;
|
||||
this->data_length = archive_block.layout.data_length;
|
||||
this->data_cursor = archive_block.layout.data_offset;
|
||||
|
||||
return find_result::ok;
|
||||
}
|
||||
|
||||
if (comparison > 0) {
|
||||
head = (midpoint + 1);
|
||||
} else {
|
||||
tail = (midpoint - 1);
|
||||
}
|
||||
}
|
||||
|
||||
return find_result::not_found;
|
||||
}
|
||||
|
||||
/**
|
||||
* Attempts to read `data.length` bytes from the file and fill `data` with it, returning the
|
||||
* number of bytes actually read or a [io_error] value to indicate an error occured.
|
||||
*/
|
||||
expected<usize, io_error> read(coral::slice<u8> const & data) override {
|
||||
if (this->data_offset < sizeof(header)) return io_error::unavailable;
|
||||
|
||||
usize const data_tail {this->data_offset + this->data_length};
|
||||
|
||||
if (!this->archive_reader->seek(coral::clamp(this->data_offset + this->data_cursor,
|
||||
this->data_offset, data_tail)).is_ok()) return io_error::unavailable;
|
||||
|
||||
expected const data_read {this->archive_reader->read(
|
||||
data.sliced(0, coral::min(data.length, data_tail - this->data_cursor)))};
|
||||
|
||||
if (data_read.is_ok()) this->data_cursor += *data_read.ok();
|
||||
|
||||
return data_read;
|
||||
}
|
||||
|
||||
/**
|
||||
* Attempts to seek to `offset` absolute position in the file, returning the new absolute
|
||||
* cursor or a [io_error] value to indicate an error occured.
|
||||
*/
|
||||
expected<u64, io_error> seek(u64 offset) override {
|
||||
if (this->data_offset < sizeof(header)) return io_error::unavailable;
|
||||
|
||||
this->data_cursor = offset;
|
||||
|
||||
return io_error::unavailable;
|
||||
}
|
||||
|
||||
/**
|
||||
* Attempts to read to read the absolute file cursor position, returning it or a [io_error]
|
||||
* value to indicate an error occured.
|
||||
*/
|
||||
expected<u64, io_error> tell() override {
|
||||
if (this->data_offset < sizeof(header)) return io_error::unavailable;
|
||||
|
||||
return this->data_cursor;
|
||||
}
|
||||
|
||||
private:
|
||||
file_reader * archive_reader {nullptr};
|
||||
|
||||
u64 data_offset {0};
|
||||
|
||||
u64 data_length {0};
|
||||
|
||||
u64 data_cursor {0};
|
||||
};
|
||||
|
||||
struct walker final : public file_walker {
|
||||
entry & archive_entry;
|
||||
|
||||
u64 index {0};
|
||||
|
||||
u64 count {0};
|
||||
|
||||
walker(entry & archive_entry) : archive_entry{archive_entry} {}
|
||||
|
||||
bool has_next() override {
|
||||
return this->index < this->count;
|
||||
}
|
||||
|
||||
expected<path, io_error> next() override {
|
||||
constexpr usize path_size {sizeof(path)};
|
||||
u8 path_buffer[path_size] {0};
|
||||
|
||||
// Read verify integrity.
|
||||
{
|
||||
expected const data_read {archive_entry.read(path_buffer)};
|
||||
|
||||
if (data_read.is_error()) return this->error();
|
||||
|
||||
switch (*data_read.ok()) {
|
||||
case path_size: break;
|
||||
case 0: return path{};
|
||||
default: return this->error();
|
||||
}
|
||||
}
|
||||
|
||||
// Verify existence of zero terminator in path.
|
||||
if (!coral::find_last(path_buffer, 0).has_value()) return this->error();
|
||||
|
||||
return path{}.joined(coral::slice{path_buffer}.as_chars());
|
||||
}
|
||||
|
||||
private:
|
||||
io_error error() {
|
||||
this->index = this->count;
|
||||
|
||||
return io_error::unavailable;
|
||||
}
|
||||
};
|
||||
|
||||
export namespace oar {
|
||||
struct archive : public fs {
|
||||
archive(fs & backing_fs, path const & archive_path) : backing_fs{backing_fs} {
|
||||
this->archive_path = archive_path;
|
||||
}
|
||||
|
||||
/**
|
||||
* See [fs::walk_files].
|
||||
*/
|
||||
void walk_files(path const & target_path, closure<void(file_walker &)> const & then) override {
|
||||
this->backing_fs.read_file(this->archive_path, [&](file_reader & archive_reader) {
|
||||
entry archive_entry{&archive_reader};
|
||||
|
||||
if (archive_entry.find(entry_kind::directory, target_path) != entry::find_result::ok) return;
|
||||
|
||||
walker archive_walker {archive_entry};
|
||||
|
||||
then(archive_walker);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* See [fs::read_file].
|
||||
*/
|
||||
void read_file(path const & file_path, closure<void(file_reader &)> const & then) override {
|
||||
this->backing_fs.read_file(this->archive_path, [&](file_reader & archive_reader) {
|
||||
entry archive_entry {&archive_reader};
|
||||
|
||||
if (archive_entry.find(entry_kind::file, file_path) != entry::find_result::ok) return;
|
||||
|
||||
then(archive_entry);
|
||||
});
|
||||
}
|
||||
|
||||
private:
|
||||
fs & backing_fs;
|
||||
|
||||
path archive_path;
|
||||
};
|
||||
|
||||
enum class [[nodiscard]] bundle_result {
|
||||
ok,
|
||||
out_of_memory,
|
||||
too_many_files,
|
||||
io_error,
|
||||
};
|
||||
|
||||
bundle_result bundle(coral::allocator & allocator, fs & output_fs,
|
||||
path const & output_path, fs & input_fs, path const & input_path) {
|
||||
|
||||
coral::small_stack<block, 64> archive_blocks {allocator};
|
||||
u64 file_count {0};
|
||||
|
||||
// Walk input dir to create blocks for all files needed.
|
||||
{
|
||||
bool has_io_error {false};
|
||||
bool is_out_of_memory {false};
|
||||
|
||||
input_fs.walk_files(input_path, [&](file_walker & walker) {
|
||||
while (walker.has_next()) {
|
||||
coral::expected const walked_path {walker.next()};
|
||||
|
||||
if (walked_path.is_error()) {
|
||||
has_io_error = true;
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (archive_blocks.push({.layout = {.path = *walked_path.ok()}}) != coral::push_result::ok) {
|
||||
is_out_of_memory = true;
|
||||
|
||||
return;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
if (has_io_error) return bundle_result::io_error;
|
||||
|
||||
if (is_out_of_memory) return bundle_result::out_of_memory;
|
||||
|
||||
if (file_count > coral::u32_max) return bundle_result::too_many_files;
|
||||
}
|
||||
|
||||
// Write header, file data, and blocks to archive.
|
||||
{
|
||||
bool has_io_error {false};
|
||||
|
||||
output_fs.write_file(output_path, [&](coral::file_writer & archive_writer) {
|
||||
header archive_header {};
|
||||
|
||||
coral::copy(archive_header.layout.signature, signature_magic);
|
||||
|
||||
// This was safety-checked during the initial file tree walk step.
|
||||
archive_header.layout.entry_count = static_cast<coral::u32>(file_count);
|
||||
|
||||
if (!archive_writer.write(archive_header.bytes).map<bool>(header::is_sizeof).ok_or(false)) {
|
||||
has_io_error = true;
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (!archive_blocks.every([&](block & archive_block) -> bool {
|
||||
bool file_read {false};
|
||||
|
||||
input_fs.read_file(archive_block.layout.path, [&](coral::file_reader & entry_reader) {
|
||||
expected const data_position {entry_reader.tell()};
|
||||
|
||||
if (data_position.is_error()) {
|
||||
has_io_error = true;
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
archive_block.layout.data_offset = *data_position.ok();
|
||||
|
||||
{
|
||||
u8 stream_buffer[4096] {0};
|
||||
expected const data_written {coral::stream(archive_writer, entry_reader, stream_buffer)};
|
||||
|
||||
if (data_written.is_error()) {
|
||||
has_io_error = true;
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
archive_block.layout.data_length = *data_written.ok();
|
||||
}
|
||||
|
||||
file_read = true;
|
||||
});
|
||||
|
||||
return file_read && (!has_io_error);
|
||||
})) return;
|
||||
|
||||
if (!archive_blocks.every([&](block const & archive_block) -> bool {
|
||||
if (!archive_writer.write(archive_block.bytes).map<bool>(block::is_sizeof).ok_or(false)) {
|
||||
has_io_error = true;
|
||||
}
|
||||
|
||||
return !has_io_error;
|
||||
})) return;
|
||||
});
|
||||
|
||||
if (has_io_error) return bundle_result::io_error;
|
||||
}
|
||||
|
||||
return bundle_result::ok;
|
||||
}
|
||||
}
|
||||
|
||||
193
source/oar/files.cpp
Normal file
193
source/oar/files.cpp
Normal file
@ -0,0 +1,193 @@
|
||||
export module oar.files;
|
||||
|
||||
import coral;
|
||||
import coral.files;
|
||||
|
||||
import oar;
|
||||
|
||||
using coral::closure;
|
||||
using coral::expected;
|
||||
using coral::file_reader;
|
||||
using coral::file_system;
|
||||
using coral::file_walker;
|
||||
using coral::io_error;
|
||||
using coral::slice;
|
||||
using coral::u64;
|
||||
|
||||
enum class [[nodiscard]] find_result {
|
||||
ok,
|
||||
not_found,
|
||||
io_unavailable,
|
||||
archive_invalid,
|
||||
};
|
||||
|
||||
struct archive_reader : public file_reader {
|
||||
archive_reader(file_reader & archive_file_reader) : archive_file_reader{archive_file_reader} {}
|
||||
|
||||
find_result find(oar::entry_path const & path) {
|
||||
this->data_offset = 0;
|
||||
this->data_length = 0;
|
||||
this->data_cursor = 0;
|
||||
|
||||
if (!this->archive_file_reader.seek(0).is_ok()) return find_result::io_unavailable;
|
||||
|
||||
oar::header_block archive_header_block;
|
||||
|
||||
if (!archive_header_block.read(this->archive_file_reader).ok().or_value(false))
|
||||
return find_result::archive_invalid;
|
||||
|
||||
// Read file table.
|
||||
u64 head {0};
|
||||
u64 tail {archive_header_block.layout.entry_count - 1};
|
||||
oar::entry_block archive_entry_block {};
|
||||
|
||||
while (head <= tail) {
|
||||
u64 const midpoint {head + ((tail - head) / 2)};
|
||||
|
||||
if (!archive_entry_block.read(this->archive_file_reader).ok().or_value(false))
|
||||
return find_result::archive_invalid;
|
||||
|
||||
coral::size const comparison {path.compare(archive_entry_block.layout.path)};
|
||||
|
||||
if (comparison == 0) {
|
||||
this->data_offset = archive_entry_block.layout.data_offset;
|
||||
this->data_length = archive_entry_block.layout.data_length;
|
||||
this->data_cursor = archive_entry_block.layout.data_offset;
|
||||
|
||||
return find_result::ok;
|
||||
}
|
||||
|
||||
if (comparison > 0) {
|
||||
head = (midpoint + 1);
|
||||
} else {
|
||||
tail = (midpoint - 1);
|
||||
}
|
||||
}
|
||||
|
||||
return find_result::not_found;
|
||||
}
|
||||
|
||||
/**
|
||||
* Attempts to read `data.length` bytes from the file and fill `data` with it, returning the
|
||||
* number of bytes actually read or a [io_error] value to indicate an error occured.
|
||||
*/
|
||||
expected<coral::usize, io_error> read(slice<coral::u8> const & data) override {
|
||||
if (this->data_offset < sizeof(oar::header_block)) return io_error::unavailable;
|
||||
|
||||
coral::usize const data_tail {this->data_offset + this->data_length};
|
||||
|
||||
if (!this->archive_file_reader.seek(coral::clamp(this->data_offset + this->data_cursor,
|
||||
this->data_offset, data_tail)).is_ok()) return io_error::unavailable;
|
||||
|
||||
expected const data_read {this->archive_file_reader.read(
|
||||
data.sliced(0, coral::min(data.length, data_tail - this->data_cursor)))};
|
||||
|
||||
if (data_read.is_ok()) this->data_cursor += *data_read.ok();
|
||||
|
||||
return data_read;
|
||||
}
|
||||
|
||||
/**
|
||||
* Attempts to seek to `offset` absolute position in the file, returning the new absolute
|
||||
* cursor or a [io_error] value to indicate an error occured.
|
||||
*/
|
||||
expected<u64, io_error> seek(u64 offset) override {
|
||||
if (this->data_offset < sizeof(oar::header_block)) return io_error::unavailable;
|
||||
|
||||
this->data_cursor = offset;
|
||||
|
||||
return io_error::unavailable;
|
||||
}
|
||||
|
||||
/**
|
||||
* Attempts to read to read the absolute file cursor position, returning it or a [io_error]
|
||||
* value to indicate an error occured.
|
||||
*/
|
||||
expected<u64, io_error> tell() override {
|
||||
if (this->data_offset < sizeof(oar::header_block)) return io_error::unavailable;
|
||||
|
||||
return this->data_cursor;
|
||||
}
|
||||
|
||||
private:
|
||||
file_reader & archive_file_reader;
|
||||
|
||||
u64 data_offset {0};
|
||||
|
||||
u64 data_length {0};
|
||||
|
||||
u64 data_cursor {0};
|
||||
};
|
||||
|
||||
struct archive_walker : public file_walker {
|
||||
archive_walker(file_reader & archive_file_reader) : archive_file_reader{archive_file_reader} {}
|
||||
|
||||
bool enumerate() override {
|
||||
// TODO: implement.
|
||||
return false;
|
||||
}
|
||||
|
||||
find_result find(oar::entry_path const & path) {
|
||||
// TODO: implement.
|
||||
return find_result::not_found;
|
||||
}
|
||||
|
||||
expected<slice<char const>, io_error> value() override {
|
||||
// TODO: implement.
|
||||
return io_error::unavailable;
|
||||
}
|
||||
|
||||
private:
|
||||
file_reader & archive_file_reader;
|
||||
|
||||
u64 files_enumerated {0};
|
||||
};
|
||||
|
||||
export namespace oar {
|
||||
struct archive_file : public file_system {
|
||||
static void read(file_system & base_file_system, slice<char const> const & archive_path, closure<void(archive_file &)> const & then) {
|
||||
if (base_file_system.query_file(archive_path).is_error()) return;
|
||||
|
||||
archive_file file {base_file_system, archive_path};
|
||||
|
||||
then(file);
|
||||
}
|
||||
|
||||
void read_file(slice<char const> const & path, closure<void(file_reader &)> const & then) override {
|
||||
this->base_file_system.read_file(this->archive_path, [&](file_reader & archive_file_reader) {
|
||||
expected const parsed_entry_path {entry_path::parse(path)};
|
||||
|
||||
if (parsed_entry_path.is_error()) return;
|
||||
|
||||
archive_reader reader {archive_file_reader};
|
||||
|
||||
if (reader.find(*parsed_entry_path.ok()) != find_result::ok) return;
|
||||
|
||||
then(reader);
|
||||
});
|
||||
}
|
||||
|
||||
void walk_files(slice<char const> const & path, closure<void(file_walker &)> const & then) override {
|
||||
this->base_file_system.read_file(this->archive_path, [&](file_reader & archive_file_reader) {
|
||||
expected const parsed_entry_path {entry_path::parse(path)};
|
||||
|
||||
if (parsed_entry_path.is_error()) return;
|
||||
|
||||
archive_walker walker {archive_file_reader};
|
||||
|
||||
if (walker.find(*parsed_entry_path.ok()) != find_result::ok) return;
|
||||
|
||||
then(walker);
|
||||
});
|
||||
}
|
||||
|
||||
private:
|
||||
file_system & base_file_system;
|
||||
|
||||
slice<char const> archive_path;
|
||||
|
||||
archive_file(file_system & base_file_system, slice<char const> const & archive_path) : base_file_system{base_file_system} {
|
||||
this->archive_path = archive_path;
|
||||
}
|
||||
};
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user