From 85dd397d62cffde650720ad7f76d1aab0c45174d Mon Sep 17 00:00:00 2001 From: lif Date: Thu, 23 Jan 2020 00:47:53 -0800 Subject: [PATCH 01/15] WIP restructure --- Cargo.lock | 17 ++++++++++ src/main.rs | 11 ++++-- src/one/mod.rs | 92 ++++++++++++++++++++++++++------------------------ 3 files changed, 73 insertions(+), 47 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 71eec31..3b6f517 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,4 +1,21 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "byteorder" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "one-rust" version = "0.1.0" +dependencies = [ + "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "prs-rust 0.1.0", +] +[[package]] +name = "prs-rust" +version = "0.1.0" + +[metadata] +"checksum byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a7c3dd8985a7111efc5c80b44e23ecdd8c007de8ade3b96595387e812b957cf5" diff --git a/src/main.rs b/src/main.rs index 886976f..858c0c1 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,8 +1,13 @@ +use std::env; use std::fs::File; pub mod one; -fn main() { - let mut infile = File::open("foo.one").unwrap(); - one::OneJod::from_archive(&mut infile).extract(); +fn main() -> std::io::Result<()> { + let args: Vec = env::args().collect(); + for arg in &args[1..] { + let mut infile = File::open(arg)?; + one::OneJod::from_archive(&mut infile)?.extract()?; + } + Ok(()) } diff --git a/src/one/mod.rs b/src/one/mod.rs index 35842d3..64d6e86 100644 --- a/src/one/mod.rs +++ b/src/one/mod.rs @@ -1,78 +1,80 @@ extern crate byteorder; extern crate prs_rust; -use self::byteorder::{ReadBytesExt, WriteBytesExt, BigEndian, LittleEndian}; +use self::byteorder::{ReadBytesExt, LittleEndian}; use self::prs_rust::prs; -use std::str; use std::fs::File; use std::io::{Read, Seek, SeekFrom, Write}; +use std::str; pub struct OneJod { read_handle: R, - header: OneJodHead, + _header: OneJodHead, entries: Vec, } -impl OneJod -where - R: Read + Seek, -{ - pub fn from_archive(mut input: R) -> OneJod { - let mut head = OneJodHead { - magic: [0; 16], - _unknown1: 0, - category: [0; 32], - file_count: 0, - reserved: [0; 128], +impl OneJod { + pub fn from_archive(mut input: R) -> std::io::Result> { + let mut magic = [0; 16]; + input.read_exact(&mut magic)?; + let _unknown1 = input.read_u32::()?; + let mut category = [0; 32]; + input.read_exact(&mut category)?; + let file_count = input.read_u32::()?; + let mut reserved = [0; 128]; + input.read_exact(&mut reserved)?; + let head = OneJodHead { + magic, + _unknown1, + category, + file_count, + reserved, }; - input.read_exact(&mut head.magic); assert_eq!(head.magic(), b"ThisIsOneFile\0\0\0"); - head._unknown1 = input.read_u32::().unwrap(); - input.read_exact(&mut head.category); - head.file_count = input.read_u32::().unwrap(); - input.read_exact(&mut head.reserved); let mut entries = Vec::new(); for i in 0..head.file_count() { - let mut entry = OneJodEntry { - _unknown1: 0, - id: 0, - size_cmp: 0, - _unknown2: 0, - size_dec: 0, - id_again: 0, - offset: 0, - file_name: [0; 188], + let _unknown1 = input.read_u32::()?; + let id = input.read_u32::()?; + let size_cmp = input.read_u32::()?; + let _unknown2 = input.read_u32::()?; + let size_dec = input.read_u32::()?; + let id_again = input.read_u32::()?; + let offset = input.read_u32::()?; + let mut file_name = [0u8; 188]; + input.read_exact(&mut file_name)?; + let entry = OneJodEntry { + _unknown1, + id, + size_cmp, + _unknown2, + size_dec, + id_again, + offset, + file_name, }; - entry._unknown1 = input.read_u32::().unwrap(); - entry.id = input.read_u32::().unwrap(); - entry.size_cmp = input.read_u32::().unwrap(); - entry._unknown2 = input.read_u32::().unwrap(); - entry.size_dec = input.read_u32::().unwrap(); - entry.id_again = input.read_u32::().unwrap(); - entry.offset = input.read_u32::().unwrap(); - input.read_exact(&mut entry.file_name); assert_eq!(entry.id(), i); entries.push(entry); } - OneJod{ + Ok(OneJod { read_handle: input, - header: head, + _header: head, entries, - } + }) } - pub fn extract(&mut self) { + pub fn extract(&mut self) -> std::io::Result<()> { for entry in &self.entries { let mut cmp_buf = vec![0u8; entry.size_cmp() as usize]; - self.read_handle.seek(SeekFrom::Start(entry.offset() as u64)); - self.read_handle.read_exact(cmp_buf.as_mut_slice()); - let mut dec_buf = prs::decompress::Decompressor::new(&cmp_buf).decompress(); + self.read_handle.seek(SeekFrom::Start(entry.offset() as u64))?; + self.read_handle.read_exact(cmp_buf.as_mut_slice())?; + let dec_buf = prs::decompress::Decompressor::new(&cmp_buf).decompress(); println!("{}", entry.file_name()); let mut file_out = File::create(entry.file_name()).unwrap(); - file_out.write_all(&dec_buf); + file_out.write_all(&dec_buf)?; } + Ok(()) } } @@ -84,6 +86,7 @@ struct OneJodHead { reserved: [u8; 128], } +#[allow(unused)] impl OneJodHead { fn magic(&self) -> &[u8] { &self.magic } fn magic_mut(&mut self) -> &mut [u8] { &mut self.magic } @@ -114,6 +117,7 @@ struct OneJodEntry { file_name: [u8; 188], } +#[allow(unused)] impl OneJodEntry { fn _unknown1(&self) -> u32 { self._unknown1 } fn set_unknown1(&mut self, n: u32) { self._unknown1 = n; } From 9d76a565062eb07ddcb84b402f6d3cb7420d2b03 Mon Sep 17 00:00:00 2001 From: lif Date: Wed, 13 Jan 2021 00:27:12 -0800 Subject: [PATCH 02/15] rename OneJod -> OneJodArchive --- src/main.rs | 2 +- src/one/mod.rs | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main.rs b/src/main.rs index 858c0c1..23434de 100644 --- a/src/main.rs +++ b/src/main.rs @@ -7,7 +7,7 @@ fn main() -> std::io::Result<()> { let args: Vec = env::args().collect(); for arg in &args[1..] { let mut infile = File::open(arg)?; - one::OneJod::from_archive(&mut infile)?.extract()?; + one::OneJodArchive::from_file(&mut infile)?.extract()?; } Ok(()) } diff --git a/src/one/mod.rs b/src/one/mod.rs index 64d6e86..bb29779 100644 --- a/src/one/mod.rs +++ b/src/one/mod.rs @@ -7,14 +7,14 @@ use std::fs::File; use std::io::{Read, Seek, SeekFrom, Write}; use std::str; -pub struct OneJod { +pub struct OneJodArchive { read_handle: R, _header: OneJodHead, entries: Vec, } -impl OneJod { - pub fn from_archive(mut input: R) -> std::io::Result> { +impl OneJodArchive { + pub fn from_file(mut input: R) -> std::io::Result> { let mut magic = [0; 16]; input.read_exact(&mut magic)?; let _unknown1 = input.read_u32::()?; @@ -57,7 +57,7 @@ impl OneJod { entries.push(entry); } - Ok(OneJod { + Ok(OneJodArchive { read_handle: input, _header: head, entries, From b4028938785da33865b4d5e6fb62f39e26c24a33 Mon Sep 17 00:00:00 2001 From: lifning <> Date: Wed, 13 Jan 2021 03:19:39 -0800 Subject: [PATCH 03/15] start rewriting to use binread --- Cargo.lock | 109 +++++++++++++++++++++++++++-- Cargo.toml | 2 + examples/onear.rs | 21 ++++++ src/lib.rs | 7 ++ src/main.rs | 13 ---- src/one/mod.rs | 174 ++++++++++------------------------------------ 6 files changed, 172 insertions(+), 154 deletions(-) create mode 100644 examples/onear.rs create mode 100644 src/lib.rs delete mode 100644 src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 3b6f517..f37da61 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,21 +1,122 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. +[[package]] +name = "binread" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "258d2d861723c05ad02956cae4878ceddb5e05b0dc2841ca73a3479ab7a46f4d" +dependencies = [ + "binread_derive", +] + +[[package]] +name = "binread_derive" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d63ec80ac459fd10ae68854c26ee1c5ab5ac93866dba8f8a95cabbfeaa79b284" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "binwrite" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42c3da1db1f384ad643e27cb6ba4f08f2a7cd8753bef8dd95c39fe45d7a474c8" +dependencies = [ + "binwrite_derive", + "paste", +] + +[[package]] +name = "binwrite_derive" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "476f17c19ce04226ba997c229ce8fb96eaef1c0e0745ec91d8df35f723aae353" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "byteorder" version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7c3dd8985a7111efc5c80b44e23ecdd8c007de8ade3b96595387e812b957cf5" [[package]] name = "one-rust" version = "0.1.0" dependencies = [ - "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "prs-rust 0.1.0", + "binread", + "binwrite", + "byteorder", + "prs-rust", +] + +[[package]] +name = "paste" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45ca20c77d80be666aef2b45486da86238fabe33e38306bd3118fe4af33fa880" +dependencies = [ + "paste-impl", + "proc-macro-hack", +] + +[[package]] +name = "paste-impl" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d95a7db200b97ef370c8e6de0088252f7e0dfff7d047a28528e47456c0fc98b6" +dependencies = [ + "proc-macro-hack", +] + +[[package]] +name = "proc-macro-hack" +version = "0.5.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" + +[[package]] +name = "proc-macro2" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71" +dependencies = [ + "unicode-xid", ] [[package]] name = "prs-rust" version = "0.1.0" -[metadata] -"checksum byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a7c3dd8985a7111efc5c80b44e23ecdd8c007de8ade3b96595387e812b957cf5" +[[package]] +name = "quote" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "991431c3519a3f36861882da93630ce66b52918dcf1b8e2fd66b397fc96f28df" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "syn" +version = "1.0.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc60a3d73ea6594cd712d830cc1f0390fd71542d8c8cd24e70cc54cdfd5e05d5" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "unicode-xid" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" diff --git a/Cargo.toml b/Cargo.toml index 2889cf4..3b6125e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,5 +4,7 @@ version = "0.1.0" authors = ["lifning "] [dependencies] +binread = "1" +binwrite = "0.2" byteorder = "1" prs-rust = { path = "../prs-rust" } diff --git a/examples/onear.rs b/examples/onear.rs new file mode 100644 index 0000000..26c7024 --- /dev/null +++ b/examples/onear.rs @@ -0,0 +1,21 @@ +extern crate one_rust; +extern crate binread; + +use std::env; +use std::fs::File; +use std::io::prelude::*; +use binread::prelude::*; +use one_rust::JodOne; + +fn main() -> Result<(), Box> { + let args: Vec = env::args().collect(); + for arg in &args[1..] { + let mut infile = File::open(arg)?; + let mut archive: JodOne = infile.read_le()?; + let entries = archive.extract(); + for (name, data) in entries { + File::create(name)?.write_all(&data)?; + } + } + Ok(()) +} diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..d6f421e --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,7 @@ +extern crate byteorder; +extern crate prs_rust; +#[macro_use] extern crate binread; +#[macro_use] extern crate binwrite; + +mod one; +pub use one::*; diff --git a/src/main.rs b/src/main.rs deleted file mode 100644 index 23434de..0000000 --- a/src/main.rs +++ /dev/null @@ -1,13 +0,0 @@ -use std::env; -use std::fs::File; - -pub mod one; - -fn main() -> std::io::Result<()> { - let args: Vec = env::args().collect(); - for arg in &args[1..] { - let mut infile = File::open(arg)?; - one::OneJodArchive::from_file(&mut infile)?.extract()?; - } - Ok(()) -} diff --git a/src/one/mod.rs b/src/one/mod.rs index bb29779..ec9c2d9 100644 --- a/src/one/mod.rs +++ b/src/one/mod.rs @@ -1,146 +1,46 @@ -extern crate byteorder; -extern crate prs_rust; +use binread::{FilePtr, NullString}; +use prs_rust::prs; +use std::collections::BTreeMap; +use std::path::PathBuf; +use std::ffi::OsStr; +use std::os::unix::ffi::OsStrExt; -use self::byteorder::{ReadBytesExt, LittleEndian}; -use self::prs_rust::prs; -use std::fs::File; -use std::io::{Read, Seek, SeekFrom, Write}; -use std::str; - -pub struct OneJodArchive { - read_handle: R, - _header: OneJodHead, - entries: Vec, +#[derive(BinRead)] +#[br(little, magic = b"ThisIsOneFile\0\0\0")] +pub struct JodOne { + _unknown1: u32, + #[br(pad_size_to = 32)] + pub category: NullString, + #[br(pad_after = 128)] + pub count: u32, + #[br(count = count)] + pub entries: Vec } -impl OneJodArchive { - pub fn from_file(mut input: R) -> std::io::Result> { - let mut magic = [0; 16]; - input.read_exact(&mut magic)?; - let _unknown1 = input.read_u32::()?; - let mut category = [0; 32]; - input.read_exact(&mut category)?; - let file_count = input.read_u32::()?; - let mut reserved = [0; 128]; - input.read_exact(&mut reserved)?; - let head = OneJodHead { - magic, - _unknown1, - category, - file_count, - reserved, - }; - assert_eq!(head.magic(), b"ThisIsOneFile\0\0\0"); +#[derive(BinRead)] +#[br(little, assert(id_again == id))] +pub struct JodOneEntry { + pub _unknown1: u32, + pub id: u32, + pub size_cmp: u32, + pub _unknown2: u32, + pub size_dec: u32, + pub id_again: u32, + #[br(count = size_cmp)] + pub data: FilePtr>, + #[br(pad_size_to = 188)] + pub name: NullString, +} - let mut entries = Vec::new(); - for i in 0..head.file_count() { - let _unknown1 = input.read_u32::()?; - let id = input.read_u32::()?; - let size_cmp = input.read_u32::()?; - let _unknown2 = input.read_u32::()?; - let size_dec = input.read_u32::()?; - let id_again = input.read_u32::()?; - let offset = input.read_u32::()?; - let mut file_name = [0u8; 188]; - input.read_exact(&mut file_name)?; - let entry = OneJodEntry { - _unknown1, - id, - size_cmp, - _unknown2, - size_dec, - id_again, - offset, - file_name, - }; - assert_eq!(entry.id(), i); - entries.push(entry); - } - - Ok(OneJodArchive { - read_handle: input, - _header: head, - entries, - }) - } - - pub fn extract(&mut self) -> std::io::Result<()> { +impl JodOne { + pub fn extract(&mut self) -> BTreeMap> { + let mut map = BTreeMap::new(); for entry in &self.entries { - let mut cmp_buf = vec![0u8; entry.size_cmp() as usize]; - self.read_handle.seek(SeekFrom::Start(entry.offset() as u64))?; - self.read_handle.read_exact(cmp_buf.as_mut_slice())?; - let dec_buf = prs::decompress::Decompressor::new(&cmp_buf).decompress(); - println!("{}", entry.file_name()); - let mut file_out = File::create(entry.file_name()).unwrap(); - file_out.write_all(&dec_buf)?; + // TODO: improve prs crate to use size_dec hint? + let dec_buf = prs::decompress::Decompressor::new(&*entry.data).decompress(); + map.insert(PathBuf::from(OsStr::from_bytes(&entry.name.0)), dec_buf); } - Ok(()) + map } } -struct OneJodHead { - magic: [u8; 16], - _unknown1: u32, - category: [u8; 32], - file_count: u32, - reserved: [u8; 128], -} - -#[allow(unused)] -impl OneJodHead { - fn magic(&self) -> &[u8] { &self.magic } - fn magic_mut(&mut self) -> &mut [u8] { &mut self.magic } - - fn _unknown1(&self) -> u32 { self._unknown1 } - fn set_unknown1(&mut self, n: u32) { self._unknown1 = n; } - - // NB: may not really be utf8? should probably be ascii, though... - fn category(&self) -> &str { str::from_utf8(&self.category).unwrap() } - fn category_mut(&mut self) -> &mut str { str::from_utf8_mut(&mut self.category).unwrap() } - fn set_category(&mut self, n: &str) { self.category.copy_from_slice(n.as_bytes()); } - - fn file_count(&self) -> u32 { self.file_count } - fn set_file_count(&mut self, n: u32) { self.file_count = n; } - - fn reserved(&self) -> &[u8] { &self.reserved } - fn reserved_mut(&mut self) -> &mut [u8] { &mut self.reserved } -} - -struct OneJodEntry { - _unknown1: u32, - id: u32, - size_cmp: u32, - _unknown2: u32, - size_dec: u32, - id_again: u32, - offset: u32, - file_name: [u8; 188], -} - -#[allow(unused)] -impl OneJodEntry { - fn _unknown1(&self) -> u32 { self._unknown1 } - fn set_unknown1(&mut self, n: u32) { self._unknown1 = n; } - - fn id(&self) -> u32 { assert_eq!(self.id, self.id_again); self.id } - fn set_id(&mut self, n: u32) { self.id = n; self.id_again = n; } - - fn size_cmp(&self) -> u32 { self.size_cmp } - fn set_size_cmp(&mut self, n: u32) { self.size_cmp = n; } - - fn _unknown2(&self) -> u32 { self._unknown2 } - fn set_unknown2(&mut self, n: u32) { self._unknown2 = n; } - - fn size_dec(&self) -> u32 { self.size_dec } - fn set_size_dec(&mut self, n: u32) { self.size_dec = n; } - - fn offset(&self) -> u32 { self.offset } - fn set_offset(&mut self, n: u32) { self.offset = n; } - - fn file_name(&self) -> &str { - let idx = self.file_name.iter().position(|x| *x == 0u8).unwrap(); - str::from_utf8(&self.file_name[0..idx]).unwrap() - } - fn file_name_mut(&mut self) -> &mut str { str::from_utf8_mut(&mut self.file_name).unwrap() } - fn set_file_name(&mut self, n: &str) { self.file_name.copy_from_slice(n.as_bytes()); } -} From 0e5cbfb772aba6fd2ed88412396d07f50817bfb4 Mon Sep 17 00:00:00 2001 From: lifning <> Date: Wed, 13 Jan 2021 03:22:08 -0800 Subject: [PATCH 04/15] gitignore Cargo.lock --- .gitignore | 1 + Cargo.lock | 122 ----------------------------------------------------- 2 files changed, 1 insertion(+), 122 deletions(-) delete mode 100644 Cargo.lock diff --git a/.gitignore b/.gitignore index 53eaa21..d0a65ef 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ /target +/Cargo.lock **/*.rs.bk diff --git a/Cargo.lock b/Cargo.lock deleted file mode 100644 index f37da61..0000000 --- a/Cargo.lock +++ /dev/null @@ -1,122 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -[[package]] -name = "binread" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "258d2d861723c05ad02956cae4878ceddb5e05b0dc2841ca73a3479ab7a46f4d" -dependencies = [ - "binread_derive", -] - -[[package]] -name = "binread_derive" -version = "1.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d63ec80ac459fd10ae68854c26ee1c5ab5ac93866dba8f8a95cabbfeaa79b284" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "binwrite" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42c3da1db1f384ad643e27cb6ba4f08f2a7cd8753bef8dd95c39fe45d7a474c8" -dependencies = [ - "binwrite_derive", - "paste", -] - -[[package]] -name = "binwrite_derive" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "476f17c19ce04226ba997c229ce8fb96eaef1c0e0745ec91d8df35f723aae353" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "byteorder" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7c3dd8985a7111efc5c80b44e23ecdd8c007de8ade3b96595387e812b957cf5" - -[[package]] -name = "one-rust" -version = "0.1.0" -dependencies = [ - "binread", - "binwrite", - "byteorder", - "prs-rust", -] - -[[package]] -name = "paste" -version = "0.1.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45ca20c77d80be666aef2b45486da86238fabe33e38306bd3118fe4af33fa880" -dependencies = [ - "paste-impl", - "proc-macro-hack", -] - -[[package]] -name = "paste-impl" -version = "0.1.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d95a7db200b97ef370c8e6de0088252f7e0dfff7d047a28528e47456c0fc98b6" -dependencies = [ - "proc-macro-hack", -] - -[[package]] -name = "proc-macro-hack" -version = "0.5.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" - -[[package]] -name = "proc-macro2" -version = "1.0.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71" -dependencies = [ - "unicode-xid", -] - -[[package]] -name = "prs-rust" -version = "0.1.0" - -[[package]] -name = "quote" -version = "1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "991431c3519a3f36861882da93630ce66b52918dcf1b8e2fd66b397fc96f28df" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "syn" -version = "1.0.58" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc60a3d73ea6594cd712d830cc1f0390fd71542d8c8cd24e70cc54cdfd5e05d5" -dependencies = [ - "proc-macro2", - "quote", - "unicode-xid", -] - -[[package]] -name = "unicode-xid" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" From 23f8959b01ea91947079b3c9eac0ada3e38fcd19 Mon Sep 17 00:00:00 2001 From: lifning <> Date: Wed, 13 Jan 2021 17:53:20 -0800 Subject: [PATCH 05/15] gitignore intellij --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index d0a65ef..a4f1f0c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ /target +/.idea /Cargo.lock **/*.rs.bk From 218489cb373b286435222ad578de5d2717f6ef23 Mon Sep 17 00:00:00 2001 From: lifning <> Date: Thu, 14 Jan 2021 00:51:08 -0800 Subject: [PATCH 06/15] directory path stuff, support uncompressed entries --- Cargo.toml | 1 + examples/onear.rs | 8 +++++++- src/lib.rs | 1 + src/one/mod.rs | 26 +++++++++++++++++++++----- 4 files changed, 30 insertions(+), 6 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 3b6125e..875ae0b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,3 +8,4 @@ binread = "1" binwrite = "0.2" byteorder = "1" prs-rust = { path = "../prs-rust" } +log = "0.4" diff --git a/examples/onear.rs b/examples/onear.rs index 26c7024..62b788c 100644 --- a/examples/onear.rs +++ b/examples/onear.rs @@ -6,6 +6,7 @@ use std::fs::File; use std::io::prelude::*; use binread::prelude::*; use one_rust::JodOne; +use std::path::PathBuf; fn main() -> Result<(), Box> { let args: Vec = env::args().collect(); @@ -13,8 +14,13 @@ fn main() -> Result<(), Box> { let mut infile = File::open(arg)?; let mut archive: JodOne = infile.read_le()?; let entries = archive.extract(); + let dir = PathBuf::from(format!("{}.d", arg)); for (name, data) in entries { - File::create(name)?.write_all(&data)?; + let full_path = dir.join(name); + if let Some(path) = full_path.parent() { + std::fs::create_dir_all(path)?; + } + File::create(full_path)?.write_all(&data)?; } } Ok(()) diff --git a/src/lib.rs b/src/lib.rs index d6f421e..e4275b3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,6 +2,7 @@ extern crate byteorder; extern crate prs_rust; #[macro_use] extern crate binread; #[macro_use] extern crate binwrite; +#[macro_use] extern crate log; mod one; pub use one::*; diff --git a/src/one/mod.rs b/src/one/mod.rs index ec9c2d9..424a50b 100644 --- a/src/one/mod.rs +++ b/src/one/mod.rs @@ -2,8 +2,9 @@ use binread::{FilePtr, NullString}; use prs_rust::prs; use std::collections::BTreeMap; use std::path::PathBuf; -use std::ffi::OsStr; -use std::os::unix::ffi::OsStrExt; +use std::ffi::OsString; +use std::os::unix::ffi::OsStringExt; +use std::fmt::Formatter; #[derive(BinRead)] #[br(little, magic = b"ThisIsOneFile\0\0\0")] @@ -32,13 +33,28 @@ pub struct JodOneEntry { pub name: NullString, } +impl std::fmt::Debug for JodOneEntry { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "JodOneEntry{{id: {}, name: {:?}, size_cmp: {}, size_dec: {}, ..}}", self.id, self.name.to_string(), self.size_cmp, self.size_dec) + } +} + impl JodOne { pub fn extract(&mut self) -> BTreeMap> { let mut map = BTreeMap::new(); for entry in &self.entries { - // TODO: improve prs crate to use size_dec hint? - let dec_buf = prs::decompress::Decompressor::new(&*entry.data).decompress(); - map.insert(PathBuf::from(OsStr::from_bytes(&entry.name.0)), dec_buf); + debug!("{:?}", entry); + let os_string = OsString::from_vec(entry.name.0.iter() + .map(|c| if char::from(*c) == '\\' { std::path::MAIN_SEPARATOR as u8 } else { *c }) + .collect()); + let key = PathBuf::from(os_string); + if entry.size_dec == 0 { + map.insert(key, entry.data.clone()); + } else { + // TODO: improve prs crate to use size_dec hint? + let dec_buf = prs::decompress::Decompressor::new(&*entry.data).decompress(); + map.insert(key, dec_buf); + } } map } From 8878eb3b3c3eea42de473a057ab293dc70ba142d Mon Sep 17 00:00:00 2001 From: lifning <> Date: Thu, 14 Jan 2021 01:01:47 -0800 Subject: [PATCH 07/15] change ownership for efficiency gains --- examples/onear.rs | 4 ++-- src/one/mod.rs | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/examples/onear.rs b/examples/onear.rs index 62b788c..839597c 100644 --- a/examples/onear.rs +++ b/examples/onear.rs @@ -12,8 +12,8 @@ fn main() -> Result<(), Box> { let args: Vec = env::args().collect(); for arg in &args[1..] { let mut infile = File::open(arg)?; - let mut archive: JodOne = infile.read_le()?; - let entries = archive.extract(); + let archive: JodOne = infile.read_le()?; + let entries = archive.unpack(); let dir = PathBuf::from(format!("{}.d", arg)); for (name, data) in entries { let full_path = dir.join(name); diff --git a/src/one/mod.rs b/src/one/mod.rs index 424a50b..6374b46 100644 --- a/src/one/mod.rs +++ b/src/one/mod.rs @@ -40,16 +40,16 @@ impl std::fmt::Debug for JodOneEntry { } impl JodOne { - pub fn extract(&mut self) -> BTreeMap> { + pub fn unpack(self) -> BTreeMap> { let mut map = BTreeMap::new(); - for entry in &self.entries { + for entry in self.entries { debug!("{:?}", entry); - let os_string = OsString::from_vec(entry.name.0.iter() - .map(|c| if char::from(*c) == '\\' { std::path::MAIN_SEPARATOR as u8 } else { *c }) + let os_string = OsString::from_vec(entry.name.0.into_iter() + .map(|c| if char::from(c) == '\\' { std::path::MAIN_SEPARATOR as u8 } else { c }) .collect()); let key = PathBuf::from(os_string); if entry.size_dec == 0 { - map.insert(key, entry.data.clone()); + map.insert(key, entry.data.into_inner()); } else { // TODO: improve prs crate to use size_dec hint? let dec_buf = prs::decompress::Decompressor::new(&*entry.data).decompress(); From 546ddbd29fb6424563b14adfee8b4fce0ed1b16b Mon Sep 17 00:00:00 2001 From: lifning <> Date: Thu, 14 Jan 2021 01:19:06 -0800 Subject: [PATCH 08/15] notes on encountered values --- src/one/mod.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/one/mod.rs b/src/one/mod.rs index 6374b46..7227947 100644 --- a/src/one/mod.rs +++ b/src/one/mod.rs @@ -9,7 +9,10 @@ use std::fmt::Formatter; #[derive(BinRead)] #[br(little, magic = b"ThisIsOneFile\0\0\0")] pub struct JodOne { + // usually 0xCC... format version maybe? + // but i've seen 0xCA in Test04/disp/Test04.one and 0xCB in Test05/disp/stg2010_sun.one _unknown1: u32, + // usually "Default" or "landData" #[br(pad_size_to = 32)] pub category: NullString, #[br(pad_after = 128)] From 5841fa4e7e2fa8f957857ceaf325e8e910378e71 Mon Sep 17 00:00:00 2001 From: lifning <> Date: Thu, 14 Jan 2021 01:32:15 -0800 Subject: [PATCH 09/15] workspace layout with prs submod --- .gitmodules | 3 +++ Cargo.toml | 13 ++----------- one-rust/Cargo.toml | 11 +++++++++++ {examples => one-rust/examples}/onear.rs | 0 {src => one-rust/src}/lib.rs | 0 {src => one-rust/src}/one/mod.rs | 0 prs-rust | 1 + 7 files changed, 17 insertions(+), 11 deletions(-) create mode 100644 .gitmodules create mode 100644 one-rust/Cargo.toml rename {examples => one-rust/examples}/onear.rs (100%) rename {src => one-rust/src}/lib.rs (100%) rename {src => one-rust/src}/one/mod.rs (100%) create mode 160000 prs-rust diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..a09e4a0 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "prs-rust"] + path = prs-rust + url = ssh://git@vvn.space:2222/lifning/prs-rust.git diff --git a/Cargo.toml b/Cargo.toml index 875ae0b..31895fa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,11 +1,2 @@ -[package] -name = "one-rust" -version = "0.1.0" -authors = ["lifning "] - -[dependencies] -binread = "1" -binwrite = "0.2" -byteorder = "1" -prs-rust = { path = "../prs-rust" } -log = "0.4" +[workspace] +members = ["one-rust", "prs-rust"] diff --git a/one-rust/Cargo.toml b/one-rust/Cargo.toml new file mode 100644 index 0000000..875ae0b --- /dev/null +++ b/one-rust/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "one-rust" +version = "0.1.0" +authors = ["lifning "] + +[dependencies] +binread = "1" +binwrite = "0.2" +byteorder = "1" +prs-rust = { path = "../prs-rust" } +log = "0.4" diff --git a/examples/onear.rs b/one-rust/examples/onear.rs similarity index 100% rename from examples/onear.rs rename to one-rust/examples/onear.rs diff --git a/src/lib.rs b/one-rust/src/lib.rs similarity index 100% rename from src/lib.rs rename to one-rust/src/lib.rs diff --git a/src/one/mod.rs b/one-rust/src/one/mod.rs similarity index 100% rename from src/one/mod.rs rename to one-rust/src/one/mod.rs diff --git a/prs-rust b/prs-rust new file mode 160000 index 0000000..f46d9e4 --- /dev/null +++ b/prs-rust @@ -0,0 +1 @@ +Subproject commit f46d9e4868af09167b6f9f8631e87b56f6e7d8d1 From 8c26a6be0675cbc62d69a6225a36a10e36ce49f8 Mon Sep 17 00:00:00 2001 From: lifning <> Date: Thu, 14 Jan 2021 01:32:43 -0800 Subject: [PATCH 10/15] ignore .one stuff used while testing --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index a4f1f0c..ecc8d50 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ /.idea /Cargo.lock **/*.rs.bk +*.one +*.one.d From aabd45b5fdbb7510d55b52fe0a02ea2fe6fe09ff Mon Sep 17 00:00:00 2001 From: lifning <> Date: Thu, 14 Jan 2021 02:28:13 -0800 Subject: [PATCH 11/15] make prs decompressor more idiomatic --- one-rust/src/one/mod.rs | 4 +--- prs-rust | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/one-rust/src/one/mod.rs b/one-rust/src/one/mod.rs index 7227947..3734224 100644 --- a/one-rust/src/one/mod.rs +++ b/one-rust/src/one/mod.rs @@ -54,12 +54,10 @@ impl JodOne { if entry.size_dec == 0 { map.insert(key, entry.data.into_inner()); } else { - // TODO: improve prs crate to use size_dec hint? - let dec_buf = prs::decompress::Decompressor::new(&*entry.data).decompress(); + let dec_buf = prs::Decompressor::new(entry.data.into_inner(), Some(entry.size_dec as usize)).decompress(); map.insert(key, dec_buf); } } map } } - diff --git a/prs-rust b/prs-rust index f46d9e4..288e8bf 160000 --- a/prs-rust +++ b/prs-rust @@ -1 +1 @@ -Subproject commit f46d9e4868af09167b6f9f8631e87b56f6e7d8d1 +Subproject commit 288e8bf588cc84cc6d23ee3476a4119ad3fb2cb5 From ccc06c81d196f5728a2e484b056598b915fe72f5 Mon Sep 17 00:00:00 2001 From: lifning <> Date: Fri, 15 Jan 2021 03:37:56 -0800 Subject: [PATCH 12/15] begin working on write support --- one-rust/Cargo.toml | 4 +-- one-rust/examples/onear.rs | 52 +++++++++++++++++++++++--------- one-rust/src/lib.rs | 4 +-- one-rust/src/one/mod.rs | 61 +++++++++++++++++++++++++++----------- prs-rust | 2 +- 5 files changed, 86 insertions(+), 37 deletions(-) diff --git a/one-rust/Cargo.toml b/one-rust/Cargo.toml index 875ae0b..718cd58 100644 --- a/one-rust/Cargo.toml +++ b/one-rust/Cargo.toml @@ -4,8 +4,8 @@ version = "0.1.0" authors = ["lifning "] [dependencies] +argh = "0.1" binread = "1" binwrite = "0.2" -byteorder = "1" -prs-rust = { path = "../prs-rust" } log = "0.4" +prs-rust = { path = "../prs-rust" } diff --git a/one-rust/examples/onear.rs b/one-rust/examples/onear.rs index 839597c..00ed007 100644 --- a/one-rust/examples/onear.rs +++ b/one-rust/examples/onear.rs @@ -1,26 +1,50 @@ -extern crate one_rust; +#[macro_use] extern crate argh; extern crate binread; +extern crate one_rust; -use std::env; use std::fs::File; use std::io::prelude::*; +use std::path::PathBuf; use binread::prelude::*; use one_rust::JodOne; -use std::path::PathBuf; + +/// Unpack .one archives from NiGHTS: Journey of Dreams +#[derive(FromArgs)] +struct Args { + #[argh(subcommand)] + action: Action, +} + +#[derive(FromArgs)] +#[argh(subcommand)] +enum Action { + ExtractFile(ExtractFile) +} + +/// Unpack and decompress the files inside a .one archive. +#[derive(FromArgs)] +#[argh(subcommand, name = "xf")] +struct ExtractFile { + /// path to the .one file + #[argh(positional)] + file: PathBuf, +} fn main() -> Result<(), Box> { - let args: Vec = env::args().collect(); - for arg in &args[1..] { - let mut infile = File::open(arg)?; - let archive: JodOne = infile.read_le()?; - let entries = archive.unpack(); - let dir = PathBuf::from(format!("{}.d", arg)); - for (name, data) in entries { - let full_path = dir.join(name); - if let Some(path) = full_path.parent() { - std::fs::create_dir_all(path)?; + let args: Args = argh::from_env(); + match args.action { + Action::ExtractFile(ex) => { + let mut infile = File::open(&ex.file)?; + let archive: JodOne = infile.read_le()?; + let entries = archive.unpack(); + let dir = PathBuf::from(format!("{}.d", ex.file.to_string_lossy())); + for (name, data) in entries { + let full_path = dir.join(name); + if let Some(path) = full_path.parent() { + std::fs::create_dir_all(path)?; + } + File::create(full_path)?.write_all(&data)?; } - File::create(full_path)?.write_all(&data)?; } } Ok(()) diff --git a/one-rust/src/lib.rs b/one-rust/src/lib.rs index e4275b3..b6e47d8 100644 --- a/one-rust/src/lib.rs +++ b/one-rust/src/lib.rs @@ -1,8 +1,8 @@ -extern crate byteorder; -extern crate prs_rust; +#![feature(fixed_size_array)] #[macro_use] extern crate binread; #[macro_use] extern crate binwrite; #[macro_use] extern crate log; +extern crate prs_rust; mod one; pub use one::*; diff --git a/one-rust/src/one/mod.rs b/one-rust/src/one/mod.rs index 3734224..6a411e0 100644 --- a/one-rust/src/one/mod.rs +++ b/one-rust/src/one/mod.rs @@ -1,20 +1,37 @@ use binread::{FilePtr, NullString}; use prs_rust::prs; -use std::collections::BTreeMap; -use std::path::PathBuf; use std::ffi::OsString; -use std::os::unix::ffi::OsStringExt; use std::fmt::Formatter; +use std::path::PathBuf; + +fn string_binread_helper(x: NullString) -> String { + unsafe { + String::from_utf8_unchecked(x.0) + } +} + +fn string_binwrite_helper(pad_to_len: usize) -> impl Fn(&String) -> Vec { + move |x: &String| { + let mut v = x.as_bytes().to_vec(); + v.resize(pad_to_len, 0); + v + } +} + +// i'm having trouble thinking atm because noisy ps3 game being played next to me #[derive(BinRead)] +#[derive(BinWrite)] #[br(little, magic = b"ThisIsOneFile\0\0\0")] +#[binwrite(little)] pub struct JodOne { // usually 0xCC... format version maybe? // but i've seen 0xCA in Test04/disp/Test04.one and 0xCB in Test05/disp/stg2010_sun.one _unknown1: u32, // usually "Default" or "landData" - #[br(pad_size_to = 32)] - pub category: NullString, + #[br(pad_size_to = 32, map = string_binread_helper)] + #[binwrite(preprocessor(string_binwrite_helper(32)))] + pub category: String, #[br(pad_after = 128)] pub count: u32, #[br(count = count)] @@ -22,7 +39,9 @@ pub struct JodOne { } #[derive(BinRead)] +#[derive(BinWrite)] #[br(little, assert(id_again == id))] +#[binwrite(little)] pub struct JodOneEntry { pub _unknown1: u32, pub id: u32, @@ -31,9 +50,11 @@ pub struct JodOneEntry { pub size_dec: u32, pub id_again: u32, #[br(count = size_cmp)] + #[binwrite(preprocessor(|x: &FilePtr>| x.ptr))] pub data: FilePtr>, - #[br(pad_size_to = 188)] - pub name: NullString, + #[br(pad_size_to = 188, map = string_binread_helper)] + #[binwrite(preprocessor(string_binwrite_helper(188)))] + pub name: String, } impl std::fmt::Debug for JodOneEntry { @@ -43,21 +64,25 @@ impl std::fmt::Debug for JodOneEntry { } impl JodOne { - pub fn unpack(self) -> BTreeMap> { - let mut map = BTreeMap::new(); - for entry in self.entries { + pub fn pack(inputs: impl Iterator)>, category: Option<&str>) -> Self { + JodOne { + _unknown1: 0xCC, + category: category.unwrap_or("Default").to_string(), + count: 0, + entries: vec![] + } + } + + pub fn unpack(self) -> impl Iterator)> { + self.entries.into_iter().map(|entry| { debug!("{:?}", entry); - let os_string = OsString::from_vec(entry.name.0.into_iter() - .map(|c| if char::from(c) == '\\' { std::path::MAIN_SEPARATOR as u8 } else { c }) - .collect()); - let key = PathBuf::from(os_string); + let key = PathBuf::from(entry.name.replace('\\', std::path::MAIN_SEPARATOR.encode_utf8(&mut [0u8; 4]))); if entry.size_dec == 0 { - map.insert(key, entry.data.into_inner()); + (key, entry.data.into_inner()) } else { let dec_buf = prs::Decompressor::new(entry.data.into_inner(), Some(entry.size_dec as usize)).decompress(); - map.insert(key, dec_buf); + (key, dec_buf) } - } - map + }) } } diff --git a/prs-rust b/prs-rust index 288e8bf..c2757ca 160000 --- a/prs-rust +++ b/prs-rust @@ -1 +1 @@ -Subproject commit 288e8bf588cc84cc6d23ee3476a4119ad3fb2cb5 +Subproject commit c2757ca1ba88ea51b386d1f48af0b9fa01829cc7 From 202f6cd640e7e53b84fd9e6f28dc20c642eab186 Mon Sep 17 00:00:00 2001 From: lifning <> Date: Sat, 16 Jan 2021 04:00:50 -0800 Subject: [PATCH 13/15] add a subcommand to describe .one archives --- one-rust/examples/onear.rs | 26 ++++++++++++++++++++++---- one-rust/src/one/mod.rs | 29 ++++++++++++++++++++++++++--- 2 files changed, 48 insertions(+), 7 deletions(-) diff --git a/one-rust/examples/onear.rs b/one-rust/examples/onear.rs index 00ed007..9713b3b 100644 --- a/one-rust/examples/onear.rs +++ b/one-rust/examples/onear.rs @@ -18,7 +18,8 @@ struct Args { #[derive(FromArgs)] #[argh(subcommand)] enum Action { - ExtractFile(ExtractFile) + ExtractFile(ExtractFile), + DescribeFile(DescribeFile), } /// Unpack and decompress the files inside a .one archive. @@ -30,14 +31,23 @@ struct ExtractFile { file: PathBuf, } +/// Describe the metadata of a .one archive. +#[derive(FromArgs)] +#[argh(subcommand, name = "tf")] +struct DescribeFile { + /// path to the .one file + #[argh(positional)] + file: PathBuf, +} + fn main() -> Result<(), Box> { let args: Args = argh::from_env(); match args.action { - Action::ExtractFile(ex) => { - let mut infile = File::open(&ex.file)?; + Action::ExtractFile(subcmd) => { + let mut infile = File::open(&subcmd.file)?; let archive: JodOne = infile.read_le()?; let entries = archive.unpack(); - let dir = PathBuf::from(format!("{}.d", ex.file.to_string_lossy())); + let dir = PathBuf::from(format!("{}.d", subcmd.file.to_string_lossy())); for (name, data) in entries { let full_path = dir.join(name); if let Some(path) = full_path.parent() { @@ -46,6 +56,14 @@ fn main() -> Result<(), Box> { File::create(full_path)?.write_all(&data)?; } } + Action::DescribeFile(subcmd) => { + let mut infile = File::open(&subcmd.file)?; + let archive: JodOne = infile.read_le()?; + println!("{:?}", archive); + for entry in archive.entries { + println!("{:?}", entry); + } + } } Ok(()) } diff --git a/one-rust/src/one/mod.rs b/one-rust/src/one/mod.rs index 6a411e0..abe188c 100644 --- a/one-rust/src/one/mod.rs +++ b/one-rust/src/one/mod.rs @@ -1,6 +1,5 @@ use binread::{FilePtr, NullString}; use prs_rust::prs; -use std::ffi::OsString; use std::fmt::Formatter; use std::path::PathBuf; @@ -46,7 +45,7 @@ pub struct JodOneEntry { pub _unknown1: u32, pub id: u32, pub size_cmp: u32, - pub _unknown2: u32, + pub is_compressed: u32, pub size_dec: u32, pub id_again: u32, #[br(count = size_cmp)] @@ -57,14 +56,38 @@ pub struct JodOneEntry { pub name: String, } +impl std::fmt::Debug for JodOne { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "JodOne{{_unknown1: 0x{:x}, category: {}, count: {}, ..}}", self._unknown1, self.category, self.count) + } +} + impl std::fmt::Debug for JodOneEntry { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "JodOneEntry{{id: {}, name: {:?}, size_cmp: {}, size_dec: {}, ..}}", self.id, self.name.to_string(), self.size_cmp, self.size_dec) + write!( + f, + "JodOneEntry{{_unknown1: 0x{:x}, id: {}, is_compressed: {}, name: {:?}, size_cmp: {}, size_dec: {}, ..}}", + self._unknown1, self.id, self.is_compressed != 0, self.name.to_string(), self.size_cmp, self.size_dec, + ) } } impl JodOne { pub fn pack(inputs: impl Iterator)>, category: Option<&str>) -> Self { + let mut entries = Vec::new(); + for (id, (path, data)) in inputs.enumerate() { + + entries.push(JodOneEntry { + _unknown1: 0, + id: id as u32, + size_cmp: data.len() as u32, + is_compressed: 0, + size_dec: 0, + id_again: id as u32, + data: FilePtr { ptr: 0, value: Some(data) }, + name: path.to_string_lossy().replace(std::path::MAIN_SEPARATOR, "\\"), + }) + } JodOne { _unknown1: 0xCC, category: category.unwrap_or("Default").to_string(), From c972b7a7c163bd723106e5df08f77ea14a55a622 Mon Sep 17 00:00:00 2001 From: lifning <> Date: Sat, 16 Jan 2021 04:55:20 -0800 Subject: [PATCH 14/15] inferring more about the format --- one-rust/src/one/mod.rs | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/one-rust/src/one/mod.rs b/one-rust/src/one/mod.rs index abe188c..64956ec 100644 --- a/one-rust/src/one/mod.rs +++ b/one-rust/src/one/mod.rs @@ -17,14 +17,21 @@ fn string_binwrite_helper(pad_to_len: usize) -> impl Fn(&String) -> Vec { } } -// i'm having trouble thinking atm because noisy ps3 game being played next to me +// stats on _unknown1 and category: +// $ find . -name \*.one -exec bash -c "onear tf "{}" > "{}".txt" \; +// $ fd one.txt | xargs cat | sed 's/count: .*/..}/' | grep 'category' | sort | uniq -c +// 1 JodOne{_unknown1: 0xca, category: Default, ..} +// 11 JodOne{_unknown1: 0xca, category: landData, ..} +// 4 JodOne{_unknown1: 0xcb, category: landData, ..} +// 988 JodOne{_unknown1: 0xcc, category: Default, ..} +// 1579 JodOne{_unknown1: 0xcc, category: landData, ..} #[derive(BinRead)] #[derive(BinWrite)] #[br(little, magic = b"ThisIsOneFile\0\0\0")] #[binwrite(little)] pub struct JodOne { - // usually 0xCC... format version maybe? + // usually 0xCC... archive creator version maybe? // but i've seen 0xCA in Test04/disp/Test04.one and 0xCB in Test05/disp/stg2010_sun.one _unknown1: u32, // usually "Default" or "landData" @@ -33,6 +40,8 @@ pub struct JodOne { pub category: String, #[br(pad_after = 128)] pub count: u32, + // always 0? is this part of the pad_after above? + pub _unknown2: u32, #[br(count = count)] pub entries: Vec } @@ -42,7 +51,6 @@ pub struct JodOne { #[br(little, assert(id_again == id))] #[binwrite(little)] pub struct JodOneEntry { - pub _unknown1: u32, pub id: u32, pub size_cmp: u32, pub is_compressed: u32, @@ -51,14 +59,14 @@ pub struct JodOneEntry { #[br(count = size_cmp)] #[binwrite(preprocessor(|x: &FilePtr>| x.ptr))] pub data: FilePtr>, - #[br(pad_size_to = 188, map = string_binread_helper)] - #[binwrite(preprocessor(string_binwrite_helper(188)))] + #[br(pad_size_to = 192, map = string_binread_helper)] + #[binwrite(preprocessor(string_binwrite_helper(192)))] pub name: String, } impl std::fmt::Debug for JodOne { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "JodOne{{_unknown1: 0x{:x}, category: {}, count: {}, ..}}", self._unknown1, self.category, self.count) + write!(f, "JodOne{{_unknown1: 0x{:x}, _unknown2: 0x{:x}, category: {}, count: {}, ..}}", self._unknown1, self._unknown2, self.category, self.count) } } @@ -66,8 +74,8 @@ impl std::fmt::Debug for JodOneEntry { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!( f, - "JodOneEntry{{_unknown1: 0x{:x}, id: {}, is_compressed: {}, name: {:?}, size_cmp: {}, size_dec: {}, ..}}", - self._unknown1, self.id, self.is_compressed != 0, self.name.to_string(), self.size_cmp, self.size_dec, + "JodOneEntry{{id: {}, is_compressed: {}, name: {:?}, size_cmp: {}, size_dec: {}, ..}}", + self.id, self.is_compressed != 0, self.name.to_string(), self.size_cmp, self.size_dec, ) } } @@ -78,7 +86,6 @@ impl JodOne { for (id, (path, data)) in inputs.enumerate() { entries.push(JodOneEntry { - _unknown1: 0, id: id as u32, size_cmp: data.len() as u32, is_compressed: 0, @@ -92,6 +99,7 @@ impl JodOne { _unknown1: 0xCC, category: category.unwrap_or("Default").to_string(), count: 0, + _unknown2: 0x20, entries: vec![] } } @@ -100,7 +108,7 @@ impl JodOne { self.entries.into_iter().map(|entry| { debug!("{:?}", entry); let key = PathBuf::from(entry.name.replace('\\', std::path::MAIN_SEPARATOR.encode_utf8(&mut [0u8; 4]))); - if entry.size_dec == 0 { + if entry.is_compressed == 0 || entry.size_dec == 0 { (key, entry.data.into_inner()) } else { let dec_buf = prs::Decompressor::new(entry.data.into_inner(), Some(entry.size_dec as usize)).decompress(); From ce190ea641842f9db1b73ff432aad61441790628 Mon Sep 17 00:00:00 2001 From: lifning <> Date: Sat, 16 Jan 2021 19:46:23 -0800 Subject: [PATCH 15/15] archive creation! --- one-rust/examples/onear.rs | 29 +++++++++++ one-rust/src/lib.rs | 4 +- one-rust/src/one/mod.rs | 102 ++++++++++++++++++++++++++----------- 3 files changed, 102 insertions(+), 33 deletions(-) diff --git a/one-rust/examples/onear.rs b/one-rust/examples/onear.rs index 9713b3b..8831a02 100644 --- a/one-rust/examples/onear.rs +++ b/one-rust/examples/onear.rs @@ -19,6 +19,7 @@ struct Args { #[argh(subcommand)] enum Action { ExtractFile(ExtractFile), + CreateFile(CreateFile), DescribeFile(DescribeFile), } @@ -40,6 +41,18 @@ struct DescribeFile { file: PathBuf, } +/// Create a .one archive from the given files. +#[derive(FromArgs)] +#[argh(subcommand, name = "cf")] +struct CreateFile { + /// path of the .one file to create + #[argh(positional)] + file: PathBuf, + /// paths of the files to pack into the archive + #[argh(positional)] + contents: Vec, +} + fn main() -> Result<(), Box> { let args: Args = argh::from_env(); match args.action { @@ -64,6 +77,22 @@ fn main() -> Result<(), Box> { println!("{:?}", entry); } } + Action::CreateFile(subcmd) => { + if subcmd.file.exists() { + return Err(format!("{:?} already exists, refusing to overwrite", subcmd.file).into()); + } + + let mut entries = Vec::new(); + for path in subcmd.contents { + let mut data = Vec::new(); + File::open(&path)?.read_to_end(&mut data)?; + entries.push((path, data)); + } + + let archive = JodOne::pack(entries, None); + let outfile = File::create(&subcmd.file)?; + archive.write(outfile)?; + } } Ok(()) } diff --git a/one-rust/src/lib.rs b/one-rust/src/lib.rs index b6e47d8..21fa636 100644 --- a/one-rust/src/lib.rs +++ b/one-rust/src/lib.rs @@ -1,6 +1,6 @@ -#![feature(fixed_size_array)] +//#![feature(fixed_size_array)] #[macro_use] extern crate binread; -#[macro_use] extern crate binwrite; +extern crate binwrite; #[macro_use] extern crate log; extern crate prs_rust; diff --git a/one-rust/src/one/mod.rs b/one-rust/src/one/mod.rs index 64956ec..5f9bd24 100644 --- a/one-rust/src/one/mod.rs +++ b/one-rust/src/one/mod.rs @@ -2,11 +2,12 @@ use binread::{FilePtr, NullString}; use prs_rust::prs; use std::fmt::Formatter; use std::path::PathBuf; +use std::mem::size_of; +use std::io::{Write, Seek}; +use binwrite::BinWrite; fn string_binread_helper(x: NullString) -> String { - unsafe { - String::from_utf8_unchecked(x.0) - } + x.into_string() } fn string_binwrite_helper(pad_to_len: usize) -> impl Fn(&String) -> Vec { @@ -17,14 +18,19 @@ fn string_binwrite_helper(pad_to_len: usize) -> impl Fn(&String) -> Vec { } } -// stats on _unknown1 and category: +// stats on _unknown1, _unknown2, and category: // $ find . -name \*.one -exec bash -c "onear tf "{}" > "{}".txt" \; -// $ fd one.txt | xargs cat | sed 's/count: .*/..}/' | grep 'category' | sort | uniq -c -// 1 JodOne{_unknown1: 0xca, category: Default, ..} -// 11 JodOne{_unknown1: 0xca, category: landData, ..} -// 4 JodOne{_unknown1: 0xcb, category: landData, ..} -// 988 JodOne{_unknown1: 0xcc, category: Default, ..} -// 1579 JodOne{_unknown1: 0xcc, category: landData, ..} +// $ fd one.txt | xargs cat | sed 's/count: .*/..}/' | grep '^JodOne{' | sort | uniq -c +// 1 JodOne{_unknown1: 0xca, _unknown2: " ", category: Default, ..} +// 11 JodOne{_unknown1: 0xca, _unknown2: " ", category: landData, ..} +// 4 JodOne{_unknown1: 0xcb, _unknown2: " ", category: landData, ..} +// 988 JodOne{_unknown1: 0xcc, _unknown2: " ", category: Default, ..} +// 1579 JodOne{_unknown1: 0xcc, _unknown2: " ", category: landData, ..} + +const JOD_ONE_MAGIC: &'static [u8; 16] = b"ThisIsOneFile\0\0\0"; + +const SIZE_OF_JOD_ONE_HEADER: usize = JOD_ONE_MAGIC.len() + 2 * size_of::() + 32 + 132; +const SIZE_OF_JOD_ONE_ENTRY: usize = 6 * size_of::() + 192; #[derive(BinRead)] #[derive(BinWrite)] @@ -34,14 +40,15 @@ pub struct JodOne { // usually 0xCC... archive creator version maybe? // but i've seen 0xCA in Test04/disp/Test04.one and 0xCB in Test05/disp/stg2010_sun.one _unknown1: u32, - // usually "Default" or "landData" + // "Default" or "landData" #[br(pad_size_to = 32, map = string_binread_helper)] #[binwrite(preprocessor(string_binwrite_helper(32)))] pub category: String, - #[br(pad_after = 128)] pub count: u32, - // always 0? is this part of the pad_after above? - pub _unknown2: u32, + // always " " + #[br(pad_size_to = 132, map = string_binread_helper)] + #[binwrite(preprocessor(string_binwrite_helper(132)))] + _unknown2: String, #[br(count = count)] pub entries: Vec } @@ -66,7 +73,7 @@ pub struct JodOneEntry { impl std::fmt::Debug for JodOne { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "JodOne{{_unknown1: 0x{:x}, _unknown2: 0x{:x}, category: {}, count: {}, ..}}", self._unknown1, self._unknown2, self.category, self.count) + write!(f, "JodOne{{_unknown1: 0x{:x}, _unknown2: {:?}, category: {}, count: {}, ..}}", self._unknown1, self._unknown2, self.category, self.count) } } @@ -81,26 +88,59 @@ impl std::fmt::Debug for JodOneEntry { } impl JodOne { - pub fn pack(inputs: impl Iterator)>, category: Option<&str>) -> Self { - let mut entries = Vec::new(); - for (id, (path, data)) in inputs.enumerate() { - - entries.push(JodOneEntry { - id: id as u32, - size_cmp: data.len() as u32, - is_compressed: 0, - size_dec: 0, - id_again: id as u32, - data: FilePtr { ptr: 0, value: Some(data) }, - name: path.to_string_lossy().replace(std::path::MAIN_SEPARATOR, "\\"), - }) + pub fn write(&self, mut writer: impl Write + Seek) -> std::io::Result<()> { + writer.write(JOD_ONE_MAGIC)?; + BinWrite::write(&self, &mut writer)?; + for entry in &self.entries { + let loc = writer.seek(std::io::SeekFrom::Current(0)).unwrap() as usize; + let padding = vec![0; entry.data.ptr as usize - loc]; + writer.write(&padding)?; + writer.write(entry.data.value.as_ref().unwrap())?; } + Ok(()) + } + + pub fn pack(inputs: impl IntoIterator)>, category: Option<&str>) -> Self { + let mut entries = Vec::new(); + for (id, (path, data)) in inputs.into_iter().enumerate() { + let cmp = prs::Compressor::new(&data, None).compress(); + let name = path.to_string_lossy().replace(std::path::MAIN_SEPARATOR, "\\"); + if cmp.len() < data.len() { + entries.push(JodOneEntry { + id: id as u32, + size_cmp: cmp.len() as u32, + is_compressed: 1, + size_dec: data.len() as u32, + id_again: id as u32, + data: FilePtr { ptr: 0, value: Some(cmp) }, + name, + }); + } else { + entries.push(JodOneEntry { + id: id as u32, + size_cmp: data.len() as u32, + is_compressed: 0, + size_dec: 0, + id_again: id as u32, + data: FilePtr { ptr: 0, value: Some(data) }, + name, + }); + } + } + + let mut loc = (SIZE_OF_JOD_ONE_HEADER + SIZE_OF_JOD_ONE_ENTRY * entries.len()) as u32; + for entry in &mut entries { + loc = ((loc + 15) / 16) * 16; // round up to 16 byte alignment + entry.data.ptr = loc; + loc += entry.data.value.as_ref().unwrap().len() as u32; + } + JodOne { _unknown1: 0xCC, category: category.unwrap_or("Default").to_string(), - count: 0, - _unknown2: 0x20, - entries: vec![] + count: entries.len() as u32, + _unknown2: " ".to_string(), + entries, } }