Merge pull request 'rework' (#1) from rework into wizeman

Reviewed-on: #1

.gitignore

@@ -1,2 +1,6 @@
 /target
+/.idea
+/Cargo.lock
 **/*.rs.bk
+*.one
+*.one.d

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "prs-rust"]
+	path = prs-rust
+	url = ssh://git@vvn.space:2222/lifning/prs-rust.git

Cargo.lock

@@ -1,4 +0,0 @@
-[[package]]
-name = "one-rust"
-version = "0.1.0"
-

Cargo.toml

@@ -1,8 +1,2 @@
-[package]
-name = "one-rust"
-version = "0.1.0"
-authors = ["lifning <lifning+git@pm.me>"]
-
-[dependencies]
-byteorder = "1"
-prs-rust = { path = "../prs-rust" }
+[workspace]
+members = ["one-rust", "prs-rust"]

one-rust/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "one-rust"
+version = "0.1.0"
+authors = ["lifning <lifning+git@pm.me>"]
+
+[dependencies]
+argh = "0.1"
+binread = "1"
+binwrite = "0.2"
+log = "0.4"
+prs-rust = { path = "../prs-rust" }

one-rust/examples/onear.rs

@@ -0,0 +1,98 @@
+#[macro_use] extern crate argh;
+extern crate binread;
+extern crate one_rust;
+
+use std::fs::File;
+use std::io::prelude::*;
+use std::path::PathBuf;
+use binread::prelude::*;
+use one_rust::JodOne;
+
+/// Unpack .one archives from NiGHTS: Journey of Dreams
+#[derive(FromArgs)]
+struct Args {
+    #[argh(subcommand)]
+    action: Action,
+}
+
+#[derive(FromArgs)]
+#[argh(subcommand)]
+enum Action {
+    ExtractFile(ExtractFile),
+    CreateFile(CreateFile),
+    DescribeFile(DescribeFile),
+}
+
+/// Unpack and decompress the files inside a .one archive.
+#[derive(FromArgs)]
+#[argh(subcommand, name = "xf")]
+struct ExtractFile {
+    /// path to the .one file
+    #[argh(positional)]
+    file: PathBuf,
+}
+
+/// Describe the metadata of a .one archive.
+#[derive(FromArgs)]
+#[argh(subcommand, name = "tf")]
+struct DescribeFile {
+    /// path to the .one file
+    #[argh(positional)]
+    file: PathBuf,
+}
+
+/// Create a .one archive from the given files.
+#[derive(FromArgs)]
+#[argh(subcommand, name = "cf")]
+struct CreateFile {
+    /// path of the .one file to create
+    #[argh(positional)]
+    file: PathBuf,
+    /// paths of the files to pack into the archive
+    #[argh(positional)]
+    contents: Vec<PathBuf>,
+}
+
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let args: Args = argh::from_env();
+    match args.action {
+        Action::ExtractFile(subcmd) => {
+            let mut infile = File::open(&subcmd.file)?;
+            let archive: JodOne = infile.read_le()?;
+            let entries = archive.unpack();
+            let dir = PathBuf::from(format!("{}.d", subcmd.file.to_string_lossy()));
+            for (name, data) in entries {
+                let full_path = dir.join(name);
+                if let Some(path) = full_path.parent() {
+                    std::fs::create_dir_all(path)?;
+                }
+                File::create(full_path)?.write_all(&data)?;
+            }
+        }
+        Action::DescribeFile(subcmd) => {
+            let mut infile = File::open(&subcmd.file)?;
+            let archive: JodOne = infile.read_le()?;
+            println!("{:?}", archive);
+            for entry in archive.entries {
+                println!("{:?}", entry);
+            }
+        }
+        Action::CreateFile(subcmd) => {
+            if subcmd.file.exists() {
+                return Err(format!("{:?} already exists, refusing to overwrite", subcmd.file).into());
+            }
+
+            let mut entries = Vec::new();
+            for path in subcmd.contents {
+                let mut data = Vec::new();
+                File::open(&path)?.read_to_end(&mut data)?;
+                entries.push((path, data));
+            }
+
+            let archive = JodOne::pack(entries, None);
+            let outfile = File::create(&subcmd.file)?;
+            archive.write(outfile)?;
+        }
+    }
+    Ok(())
+}

one-rust/src/lib.rs

@@ -0,0 +1,8 @@
+//#![feature(fixed_size_array)]
+#[macro_use] extern crate binread;
+extern crate binwrite;
+#[macro_use] extern crate log;
+extern crate prs_rust;
+
+mod one;
+pub use one::*;

one-rust/src/one/mod.rs

@@ -0,0 +1,159 @@
+use binread::{FilePtr, NullString};
+use prs_rust::prs;
+use std::fmt::Formatter;
+use std::path::PathBuf;
+use std::mem::size_of;
+use std::io::{Write, Seek};
+use binwrite::BinWrite;
+
+fn string_binread_helper(x: NullString) -> String {
+    x.into_string()
+}
+
+fn string_binwrite_helper(pad_to_len: usize) -> impl Fn(&String) -> Vec<u8> {
+    move |x: &String| {
+        let mut v = x.as_bytes().to_vec();
+        v.resize(pad_to_len, 0);
+        v
+    }
+}
+
+// stats on _unknown1, _unknown2, and category:
+// $ find . -name \*.one -exec bash -c "onear tf "{}" > "{}".txt" \;
+// $ fd one.txt | xargs cat | sed 's/count: .*/..}/' | grep '^JodOne{' | sort | uniq -c
+//       1 JodOne{_unknown1: 0xca, _unknown2: " ", category: Default, ..}
+//      11 JodOne{_unknown1: 0xca, _unknown2: " ", category: landData, ..}
+//       4 JodOne{_unknown1: 0xcb, _unknown2: " ", category: landData, ..}
+//     988 JodOne{_unknown1: 0xcc, _unknown2: " ", category: Default, ..}
+//    1579 JodOne{_unknown1: 0xcc, _unknown2: " ", category: landData, ..}
+
+const JOD_ONE_MAGIC: &'static [u8; 16] = b"ThisIsOneFile\0\0\0";
+
+const SIZE_OF_JOD_ONE_HEADER: usize = JOD_ONE_MAGIC.len() + 2 * size_of::<u32>() + 32 + 132;
+const SIZE_OF_JOD_ONE_ENTRY: usize = 6 * size_of::<u32>() + 192;
+
+#[derive(BinRead)]
+#[derive(BinWrite)]
+#[br(little, magic = b"ThisIsOneFile\0\0\0")]
+#[binwrite(little)]
+pub struct JodOne {
+    // usually 0xCC... archive creator version maybe?
+    // but i've seen 0xCA in Test04/disp/Test04.one and 0xCB in Test05/disp/stg2010_sun.one
+    _unknown1: u32,
+    // "Default" or "landData"
+    #[br(pad_size_to = 32, map = string_binread_helper)]
+    #[binwrite(preprocessor(string_binwrite_helper(32)))]
+    pub category: String,
+    pub count: u32,
+    // always " "
+    #[br(pad_size_to = 132, map = string_binread_helper)]
+    #[binwrite(preprocessor(string_binwrite_helper(132)))]
+    _unknown2: String,
+    #[br(count = count)]
+    pub entries: Vec<JodOneEntry>
+}
+
+#[derive(BinRead)]
+#[derive(BinWrite)]
+#[br(little, assert(id_again == id))]
+#[binwrite(little)]
+pub struct JodOneEntry {
+    pub id: u32,
+    pub size_cmp: u32,
+    pub is_compressed: u32,
+    pub size_dec: u32,
+    pub id_again: u32,
+    #[br(count = size_cmp)]
+    #[binwrite(preprocessor(|x: &FilePtr<u32, Vec<u8>>| x.ptr))]
+    pub data: FilePtr<u32, Vec<u8>>,
+    #[br(pad_size_to = 192, map = string_binread_helper)]
+    #[binwrite(preprocessor(string_binwrite_helper(192)))]
+    pub name: String,
+}
+
+impl std::fmt::Debug for JodOne {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        write!(f, "JodOne{{_unknown1: 0x{:x}, _unknown2: {:?}, category: {}, count: {}, ..}}", self._unknown1, self._unknown2, self.category, self.count)
+    }
+}
+
+impl std::fmt::Debug for JodOneEntry {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "JodOneEntry{{id: {}, is_compressed: {}, name: {:?}, size_cmp: {}, size_dec: {}, ..}}",
+            self.id, self.is_compressed != 0, self.name.to_string(), self.size_cmp, self.size_dec,
+        )
+    }
+}
+
+impl JodOne {
+    pub fn write(&self, mut writer: impl Write + Seek) -> std::io::Result<()> {
+        writer.write(JOD_ONE_MAGIC)?;
+        BinWrite::write(&self, &mut writer)?;
+        for entry in &self.entries {
+            let loc = writer.seek(std::io::SeekFrom::Current(0)).unwrap() as usize;
+            let padding = vec![0; entry.data.ptr as usize - loc];
+            writer.write(&padding)?;
+            writer.write(entry.data.value.as_ref().unwrap())?;
+        }
+        Ok(())
+    }
+
+    pub fn pack(inputs: impl IntoIterator<Item = (PathBuf, Vec<u8>)>, category: Option<&str>) -> Self {
+        let mut entries = Vec::new();
+        for (id, (path, data)) in inputs.into_iter().enumerate() {
+            let cmp = prs::Compressor::new(&data, None).compress();
+            let name = path.to_string_lossy().replace(std::path::MAIN_SEPARATOR, "\\");
+            if cmp.len() < data.len() {
+                entries.push(JodOneEntry {
+                    id: id as u32,
+                    size_cmp: cmp.len() as u32,
+                    is_compressed: 1,
+                    size_dec: data.len() as u32,
+                    id_again: id as u32,
+                    data: FilePtr { ptr: 0, value: Some(cmp) },
+                    name,
+                });
+            } else {
+                entries.push(JodOneEntry {
+                    id: id as u32,
+                    size_cmp: data.len() as u32,
+                    is_compressed: 0,
+                    size_dec: 0,
+                    id_again: id as u32,
+                    data: FilePtr { ptr: 0, value: Some(data) },
+                    name,
+                });
+            }
+        }
+
+        let mut loc = (SIZE_OF_JOD_ONE_HEADER + SIZE_OF_JOD_ONE_ENTRY * entries.len()) as u32;
+        for entry in &mut entries {
+            loc = ((loc + 15) / 16) * 16; // round up to 16 byte alignment
+            entry.data.ptr = loc;
+            loc += entry.data.value.as_ref().unwrap().len() as u32;
+        }
+
+        JodOne {
+            _unknown1: 0xCC,
+            category: category.unwrap_or("Default").to_string(),
+            count: entries.len() as u32,
+            _unknown2: " ".to_string(),
+            entries,
+        }
+    }
+
+    pub fn unpack(self) -> impl Iterator<Item = (PathBuf, Vec<u8>)> {
+        self.entries.into_iter().map(|entry| {
+            debug!("{:?}", entry);
+            let key = PathBuf::from(entry.name.replace('\\', std::path::MAIN_SEPARATOR.encode_utf8(&mut [0u8; 4])));
+            if entry.is_compressed == 0 || entry.size_dec == 0 {
+                (key, entry.data.into_inner())
+            } else {
+                let dec_buf = prs::Decompressor::new(entry.data.into_inner(), Some(entry.size_dec as usize)).decompress();
+                (key, dec_buf)
+            }
+        })
+    }
+}

prs-rust

@@ -0,0 +1 @@
+Subproject commit c2757ca1ba88ea51b386d1f48af0b9fa01829cc7

src/main.rs

@@ -1,8 +0,0 @@
-use std::fs::File;
-
-pub mod one;
-
-fn main() {
-    let mut infile = File::open("foo.one").unwrap();
-    one::OneJod::from_archive(&mut infile).extract();
-}

src/one/mod.rs

@@ -1,142 +0,0 @@
-extern crate byteorder;
-extern crate prs_rust;
-
-use self::byteorder::{ReadBytesExt, WriteBytesExt, BigEndian, LittleEndian};
-use self::prs_rust::prs;
-use std::str;
-use std::fs::File;
-use std::io::{Read, Seek, SeekFrom, Write};
-
-pub struct OneJod<R: Read + Seek> {
-    read_handle: R,
-    header: OneJodHead,
-    entries: Vec<OneJodEntry>,
-}
-
-impl<R> OneJod<R>
-where
-    R: Read + Seek,
-{
-    pub fn from_archive(mut input: R) -> OneJod<R> {
-        let mut head = OneJodHead {
-            magic: [0; 16],
-            _unknown1: 0,
-            category: [0; 32],
-            file_count: 0,
-            reserved: [0; 128],
-        };
-        input.read_exact(&mut head.magic);
-        assert_eq!(head.magic(), b"ThisIsOneFile\0\0\0");
-        head._unknown1 = input.read_u32::<byteorder::LittleEndian>().unwrap();
-        input.read_exact(&mut head.category);
-        head.file_count = input.read_u32::<byteorder::LittleEndian>().unwrap();
-        input.read_exact(&mut head.reserved);
-
-        let mut entries = Vec::new();
-        for i in 0..head.file_count() {
-            let mut entry = OneJodEntry {
-                _unknown1: 0,
-                id: 0,
-                size_cmp: 0,
-                _unknown2: 0,
-                size_dec: 0,
-                id_again: 0,
-                offset: 0,
-                file_name: [0; 188],
-            };
-            entry._unknown1 = input.read_u32::<byteorder::LittleEndian>().unwrap();
-            entry.id = input.read_u32::<byteorder::LittleEndian>().unwrap();
-            entry.size_cmp = input.read_u32::<byteorder::LittleEndian>().unwrap();
-            entry._unknown2 = input.read_u32::<byteorder::LittleEndian>().unwrap();
-            entry.size_dec = input.read_u32::<byteorder::LittleEndian>().unwrap();
-            entry.id_again = input.read_u32::<byteorder::LittleEndian>().unwrap();
-            entry.offset = input.read_u32::<byteorder::LittleEndian>().unwrap();
-            input.read_exact(&mut entry.file_name);
-            assert_eq!(entry.id(), i);
-            entries.push(entry);
-        }
-
-        OneJod{
-            read_handle: input,
-            header: head,
-            entries,
-        }
-    }
-
-    pub fn extract(&mut self) {
-        for entry in &self.entries {
-            let mut cmp_buf = vec![0u8; entry.size_cmp() as usize];
-            self.read_handle.seek(SeekFrom::Start(entry.offset() as u64));
-            self.read_handle.read_exact(cmp_buf.as_mut_slice());
-            let mut dec_buf = prs::decompress::Decompressor::new(&cmp_buf).decompress();
-            println!("{}", entry.file_name());
-            let mut file_out = File::create(entry.file_name()).unwrap();
-            file_out.write_all(&dec_buf);
-        }
-    }
-}
-
-struct OneJodHead {
-    magic: [u8; 16],
-    _unknown1: u32,
-    category: [u8; 32],
-    file_count: u32,
-    reserved: [u8; 128],
-}
-
-impl OneJodHead {
-    fn magic(&self) -> &[u8] { &self.magic }
-    fn magic_mut(&mut self) -> &mut [u8] { &mut self.magic }
-
-    fn _unknown1(&self) -> u32 { self._unknown1 }
-    fn set_unknown1(&mut self, n: u32) { self._unknown1 = n; }
-
-    // NB: may not really be utf8?  should probably be ascii, though...
-    fn category(&self) -> &str { str::from_utf8(&self.category).unwrap() }
-    fn category_mut(&mut self) -> &mut str { str::from_utf8_mut(&mut self.category).unwrap() }
-    fn set_category(&mut self, n: &str) { self.category.copy_from_slice(n.as_bytes()); }
-
-    fn file_count(&self) -> u32 { self.file_count }
-    fn set_file_count(&mut self, n: u32) { self.file_count = n; }
-
-    fn reserved(&self) -> &[u8] { &self.reserved }
-    fn reserved_mut(&mut self) -> &mut [u8] { &mut self.reserved }
-}
-
-struct OneJodEntry {
-    _unknown1: u32,
-    id: u32,
-    size_cmp: u32,
-    _unknown2: u32,
-    size_dec: u32,
-    id_again: u32,
-    offset: u32,
-    file_name: [u8; 188],
-}
-
-impl OneJodEntry {
-    fn _unknown1(&self) -> u32 { self._unknown1 }
-    fn set_unknown1(&mut self, n: u32) { self._unknown1 = n; }
-
-    fn id(&self) -> u32 { assert_eq!(self.id, self.id_again); self.id }
-    fn set_id(&mut self, n: u32) { self.id = n; self.id_again = n; }
-
-    fn size_cmp(&self) -> u32 { self.size_cmp }
-    fn set_size_cmp(&mut self, n: u32) { self.size_cmp = n; }
-
-    fn _unknown2(&self) -> u32 { self._unknown2 }
-    fn set_unknown2(&mut self, n: u32) { self._unknown2 = n; }
-
-    fn size_dec(&self) -> u32 { self.size_dec }
-    fn set_size_dec(&mut self, n: u32) { self.size_dec = n; }
-
-    fn offset(&self) -> u32 { self.offset }
-    fn set_offset(&mut self, n: u32) { self.offset = n; }
-
-    fn file_name(&self) -> &str {
-        let idx = self.file_name.iter().position(|x| *x == 0u8).unwrap();
-        str::from_utf8(&self.file_name[0..idx]).unwrap()
-    }
-    fn file_name_mut(&mut self) -> &mut str { str::from_utf8_mut(&mut self.file_name).unwrap() }
-    fn set_file_name(&mut self, n: &str) { self.file_name.copy_from_slice(n.as_bytes()); }
-}