diff --git a/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataEntity.java b/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataEntity.java index 864d3044..829ee9c2 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataEntity.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/entities/data/DataEntity.java @@ -48,6 +48,16 @@ public void addAuthorId(String id) { this.addIdProperty("author", id); } + /** + * Gets the path of this entity. + *

+ * When reading a crate, this will point to the entity's file within the crate. + * When creating a new entity or crate, it might point to a file outside the crate, + * as set by {@link DataEntityBuilder#setLocation(Path)}. + * Such a file will be copied into the crate when writing. + * + * @return the path to the file this entity represents. + */ @JsonIgnore public Path getPath() { return path; diff --git a/src/main/java/edu/kit/datamanager/ro_crate/reader/CrateReader.java b/src/main/java/edu/kit/datamanager/ro_crate/reader/CrateReader.java index 15cae146..72d30e79 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/reader/CrateReader.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/reader/CrateReader.java @@ -22,6 +22,7 @@ import java.nio.file.Path; import java.util.*; import java.util.stream.Collectors; +import java.util.stream.Stream; import java.util.stream.StreamSupport; /** @@ -142,6 +143,7 @@ private RoCrate rebuildCrate(ObjectNode metadataJson, File files, HashSet { + System.out.println("Found file for: " + file.toString()); usedFiles.add(file.getPath()); builder.setLocationWithExceptions(file.toPath()) .setId(file.getName()); @@ -237,15 +239,24 @@ protected String unpackId(JsonNode node) { } protected Optional checkFolderHasFile(String filepathOrId, File folder) { + System.out.println("DEBUG: " + Arrays.toString(folder.listFiles())); if (IdentifierUtils.isUrl(filepathOrId)) { return Optional.empty(); } - return IdentifierUtils.decode(filepathOrId) - .map(decoded -> folder.toPath().resolve(decoded).normalize()) + System.out.println( "DEBUG: " + + IdentifierUtils.decode(filepathOrId) + .map(decoded -> folder.toPath().resolve(decoded).normalize()) + .filter(resolved -> resolved.startsWith(folder.toPath())) + .map(Path::toFile) + .map(File::exists) + ); + return Stream.of(IdentifierUtils.decode(filepathOrId).orElse(filepathOrId), filepathOrId) + .map(filename -> folder.toPath().resolve(filename).normalize().toAbsolutePath()) // defence-in-depth: ensure we are still inside the crate folder .filter(resolved -> resolved.startsWith(folder.toPath())) .map(Path::toFile) - .filter(File::exists); + .filter(File::exists) + .findFirst(); } /** diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/WriteFolderStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/WriteFolderStrategy.java index 8a4844ed..a694c0ca 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/WriteFolderStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/WriteFolderStrategy.java @@ -5,6 +5,7 @@ import edu.kit.datamanager.ro_crate.Crate; import edu.kit.datamanager.ro_crate.entities.data.DataEntity; import edu.kit.datamanager.ro_crate.objectmapper.MyObjectMapper; +import edu.kit.datamanager.ro_crate.special.IdentifierUtils; import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -64,16 +65,18 @@ public void save(Crate crate, String destination) throws IOException { } } for (DataEntity dataEntity : crate.getAllDataEntities()) { - savetoFile(dataEntity, file); + saveToFile(dataEntity, file); } } - private void savetoFile(DataEntity entity, File file) throws IOException { + private void saveToFile(DataEntity entity, File file) throws IOException { if (entity.getPath() != null) { + String id = entity.getId(); + String filename = IdentifierUtils.decode(id).orElse(id); if (entity.getPath().toFile().isDirectory()) { - FileUtils.copyDirectory(entity.getPath().toFile(), file.toPath().resolve(entity.getId()).toFile()); + FileUtils.copyDirectory(entity.getPath().toFile(), file.toPath().resolve(filename).toFile()); } else { - FileUtils.copyFile(entity.getPath().toFile(), file.toPath().resolve(entity.getId()).toFile()); + FileUtils.copyFile(entity.getPath().toFile(), file.toPath().resolve(filename).toFile()); } } } diff --git a/src/main/java/edu/kit/datamanager/ro_crate/writer/WriteZipStreamStrategy.java b/src/main/java/edu/kit/datamanager/ro_crate/writer/WriteZipStreamStrategy.java index 33572db2..5216ccb5 100644 --- a/src/main/java/edu/kit/datamanager/ro_crate/writer/WriteZipStreamStrategy.java +++ b/src/main/java/edu/kit/datamanager/ro_crate/writer/WriteZipStreamStrategy.java @@ -15,6 +15,7 @@ import java.util.UUID; import edu.kit.datamanager.ro_crate.preview.CratePreview; +import edu.kit.datamanager.ro_crate.special.IdentifierUtils; import edu.kit.datamanager.ro_crate.util.FileSystemUtil; import edu.kit.datamanager.ro_crate.util.ZipStreamUtil; import net.lingala.zip4j.io.outputstream.ZipOutputStream; @@ -154,16 +155,18 @@ private void saveToStream(DataEntity entity, ZipOutputStream zipStream, String p } boolean isDirectory = entity.getPath().toFile().isDirectory(); + String id = entity.getId(); + String filename = IdentifierUtils.decode(id).orElse(id); if (isDirectory) { ZipStreamUtil.addFolderToZipStream( zipStream, entity.getPath().toFile(), - prefix + entity.getId()); + prefix + filename); } else { ZipStreamUtil.addFileToZipStream( zipStream, entity.getPath().toFile(), - prefix + entity.getId()); + prefix + filename); } } } diff --git a/src/test/java/edu/kit/datamanager/ro_crate/crate/ReadAndWriteTest.java b/src/test/java/edu/kit/datamanager/ro_crate/crate/ReadAndWriteTest.java index ca742ded..7688e09f 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/crate/ReadAndWriteTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/crate/ReadAndWriteTest.java @@ -1,42 +1,61 @@ package edu.kit.datamanager.ro_crate.crate; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + import edu.kit.datamanager.ro_crate.Crate; import edu.kit.datamanager.ro_crate.HelpFunctions; import edu.kit.datamanager.ro_crate.RoCrate; +import edu.kit.datamanager.ro_crate.entities.data.DataEntity; +import edu.kit.datamanager.ro_crate.entities.data.FileEntity; import edu.kit.datamanager.ro_crate.preview.StaticPreview; import edu.kit.datamanager.ro_crate.reader.CrateReader; import edu.kit.datamanager.ro_crate.reader.Readers; - +import edu.kit.datamanager.ro_crate.special.IdentifierUtils; +import edu.kit.datamanager.ro_crate.writer.CrateWriter; import edu.kit.datamanager.ro_crate.writer.Writers; -import org.apache.commons.io.FileUtils; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; - import java.io.IOException; import java.nio.charset.Charset; +import java.nio.file.Files; import java.nio.file.Path; - -import static org.junit.jupiter.api.Assertions.assertEquals; - +import java.util.List; +import java.util.UUID; +import org.apache.commons.io.FileUtils; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; class ReadAndWriteTest { @Test void testReadingAndWriting(@TempDir Path path) throws IOException { Path htmlFile = path.resolve("htmlFile.html"); - FileUtils.writeStringToFile(htmlFile.toFile(), "useful file", Charset.defaultCharset()); + FileUtils.writeStringToFile( + htmlFile.toFile(), + "useful file", + Charset.defaultCharset() + ); Path htmlDir = path.resolve("dir"); Path fileInDir = htmlDir.resolve("file.html"); - FileUtils.writeStringToFile(fileInDir.toFile(), "fileN2", Charset.defaultCharset()); + FileUtils.writeStringToFile( + fileInDir.toFile(), + "fileN2", + Charset.defaultCharset() + ); - RoCrate crate = new RoCrate.RoCrateBuilder("name", "description", "2024", "https://creativecommons.org/licenses/by-nc-sa/3.0/au/") - .setPreview(new StaticPreview(htmlFile.toFile(), htmlDir.toFile())) - .build(); + RoCrate crate = new RoCrate.RoCrateBuilder( + "name", + "description", + "2024", + "https://creativecommons.org/licenses/by-nc-sa/3.0/au/" + ) + .setPreview(new StaticPreview(htmlFile.toFile(), htmlDir.toFile())) + .build(); Path writeDir = path.resolve("crate"); - Writers.newFolderWriter() - .save(crate, writeDir.toAbsolutePath().toString()); + Writers.newFolderWriter().save(crate, writeDir.toAbsolutePath().toString()); CrateReader reader = Readers.newFolderReader(); Crate newCrate = reader.readCrate(writeDir.toAbsolutePath().toString()); @@ -51,8 +70,215 @@ void testReadingAndWriting(@TempDir Path path) throws IOException { @Test void testReadCrateWithHasPartHierarchy() throws IOException { CrateReader reader = Readers.newFolderReader(); - RoCrate crate = reader.readCrate(ReadAndWriteTest.class.getResource("/crates/hasPartHierarchy").getPath()); + RoCrate crate = reader.readCrate( + ReadAndWriteTest.class.getResource("/crates/hasPartHierarchy").getPath() + ); assertEquals(1, crate.getAllContextualEntities().size()); assertEquals(6, crate.getAllDataEntities().size()); } + + @Test + void testEncodedIdsFindTheirPaths(@TempDir Path tempDir) throws IOException { + RoCrate.RoCrateBuilder builder = new RoCrate.RoCrateBuilder(); + { + FileEntity.FileEntityBuilder dataEntityBuilder = + new FileEntity.FileEntityBuilder(); + dataEntityBuilder.setId("id 1"); + dataEntityBuilder.addTypes(List.of("File")); + UUID uuid = UUID.randomUUID(); + Path path = tempDir.resolve(uuid.toString()); + Files.writeString(path, "File"); + dataEntityBuilder.setLocation(path); + + builder.addDataEntity(dataEntityBuilder.build()); + } + { + FileEntity.FileEntityBuilder dataEntityBuilder = + new FileEntity.FileEntityBuilder(); + dataEntityBuilder.setId("id\uD83E\uDD791"); + dataEntityBuilder.addTypes(List.of("File")); + UUID uuid = UUID.randomUUID(); + Path path = tempDir.resolve(uuid.toString()); + Files.writeString(path, "File"); + dataEntityBuilder.setLocation(path); + + builder.addDataEntity(dataEntityBuilder.build()); + } + { + FileEntity.FileEntityBuilder dataEntityBuilder = + new FileEntity.FileEntityBuilder(); + dataEntityBuilder.setId("id|1"); + dataEntityBuilder.addTypes(List.of("File")); + UUID uuid = UUID.randomUUID(); + Path path = tempDir.resolve(uuid.toString()); + Files.writeString(path, "File"); + dataEntityBuilder.setLocation(path); + + builder.addDataEntity(dataEntityBuilder.build()); + } + Path location = tempDir.resolve("out"); + { + RoCrate crate = builder.build(); + CrateWriter writer = Writers.newFolderWriter(); + writer.save(crate, location.toString()); + } + { + CrateReader roCrateReader = Readers.newFolderReader(); + + RoCrate roCrate = roCrateReader.readCrate( + location.toString() + ); + for (DataEntity dataEntity : roCrate.getAllDataEntities()) { + System.out.println(dataEntity.getId() + ": " + dataEntity.getPath()); + } + for (DataEntity dataEntity : roCrate.getAllDataEntities()) { + assertNotNull( + dataEntity.getPath(), + "Path of ID: " + dataEntity.getId() + ); + } + } + } + + /** + * Test we detect files which use the encoded IDs as filename, + * as well as ones which use the decoded filename. + */ + @Test + void testDetectingEncodedFileNames(@TempDir Path tempDir) throws IOException { + // This is how we add the id. But the space will be encoded + String id = "id 42"; + // This is how we get it out (the encoded id as it will exist in the crate) + String idEncoded = IdentifierUtils.encode(id).orElseThrow(); + + RoCrate.RoCrateBuilder builder = new RoCrate.RoCrateBuilder(); + + { + // Add file entity without a file with id + FileEntity.FileEntityBuilder fileEntityBuilder = + new FileEntity.FileEntityBuilder(); + fileEntityBuilder.setId(id); + fileEntityBuilder.addTypes(List.of("File")); + + builder.addDataEntity(fileEntityBuilder.build()); + } + + Path cratepath1 = tempDir.resolve("test1"); + { + Writers.newFolderWriter().save(builder.build(), cratepath1.toString()); + // add file manually (decoded id) + Path filepath = cratepath1.resolve(id); + Files.writeString(filepath, "File"); + } + + Path cratepath2 = tempDir.resolve("test2"); + { + Writers.newFolderWriter().save(builder.build(), cratepath2.toString()); + // add file manually (encoded id) + Path filepath = cratepath2.resolve(idEncoded); + Files.writeString(filepath, "File"); + } + + { + RoCrate crate = Readers.newFolderReader().readCrate( + cratepath1.toString() + ); + + DataEntity entity = crate.getDataEntityById(idEncoded); + assertEquals(idEncoded, entity.getId()); + + Path filepath = entity.getPath(); + assertNotNull(filepath); + assertTrue(Files.exists(filepath)); + assertEquals(id, filepath.getFileName().toString()); + } + + { + RoCrate crate = Readers.newFolderReader().readCrate( + cratepath2.toString() + ); + + DataEntity entity = crate.getDataEntityById(idEncoded); + assertEquals(idEncoded, entity.getId()); + + Path filepath = entity.getPath(); + assertNotNull(filepath); + assertTrue(Files.exists(filepath)); + assertEquals(idEncoded, filepath.getFileName().toString()); + } + } + + @Test + void testFilenamesAreSelfHealing(@TempDir Path tempDir) throws IOException { + // This is how we add the id. But the space will be encoded + String id = "id 42"; + // This is how we get it out (the encoded id as it will exist in the crate) + String idEncoded = IdentifierUtils.encode(id).orElseThrow(); + + RoCrate.RoCrateBuilder builder = new RoCrate.RoCrateBuilder(); + + // add dummy file + Path filepath_outside = tempDir.resolve("someFile.txt"); + Files.writeString(filepath_outside, "File"); + + { + // Add file entity without a file with id + FileEntity.FileEntityBuilder fileEntityBuilder = + new FileEntity.FileEntityBuilder(); + fileEntityBuilder.setId(id); + fileEntityBuilder.addTypes(List.of("File")); + fileEntityBuilder.setLocation(filepath_outside); + + builder.addDataEntity(fileEntityBuilder.build()); + } + + Path cratepath = tempDir.resolve("test1"); + { + Writers.newFolderWriter().save(builder.build(), cratepath.toString()); + Path currentFilePath = cratepath.resolve(id); + assertTrue(currentFilePath.toFile().exists()); + // swap file names + Path newFilePath = cratepath.resolve(idEncoded); + Files.move(currentFilePath, newFilePath); + assertFalse(currentFilePath.toFile().exists()); + assertTrue(newFilePath.toFile().exists()); + } + + { + RoCrate crate = Readers.newFolderReader().readCrate( + cratepath.toString() + ); + + DataEntity entity = crate.getDataEntityById(idEncoded); + assertEquals(idEncoded, entity.getId()); + + // Even if a file's name is encoded, the path will work as expected + Path filepath = entity.getPath(); + assertNotNull(filepath); + assertTrue(Files.exists(filepath)); + assertEquals(idEncoded, filepath.getFileName().toString()); + } + + // When saving the crate again, the file will be renamed to the decoded id + Path cratepath2 = tempDir.resolve("test2"); + { + Writers.newFolderWriter().save(builder.build(), cratepath2.toString()); + } + + { + RoCrate crate = Readers.newFolderReader().readCrate( + cratepath2.toString() + ); + + DataEntity entity = crate.getDataEntityById(idEncoded); + assertEquals(idEncoded, entity.getId()); + + // The filename is now decoded to the original id + Path filepath = entity.getPath(); + assertNotNull(filepath); + assertTrue(Files.exists(filepath)); + assertEquals(id, filepath.getFileName().toString()); + } + + } } diff --git a/src/test/java/edu/kit/datamanager/ro_crate/writer/CommonWriterTest.java b/src/test/java/edu/kit/datamanager/ro_crate/writer/CommonWriterTest.java index 0eae96c2..3f67f4f2 100644 --- a/src/test/java/edu/kit/datamanager/ro_crate/writer/CommonWriterTest.java +++ b/src/test/java/edu/kit/datamanager/ro_crate/writer/CommonWriterTest.java @@ -4,6 +4,7 @@ import edu.kit.datamanager.ro_crate.RoCrate; import edu.kit.datamanager.ro_crate.entities.data.DataSetEntity; +import edu.kit.datamanager.ro_crate.entities.data.FileEntity; import org.apache.commons.io.FileUtils; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -34,6 +35,7 @@ default void testFilesBeingAdjusted(@TempDir Path tempDir) throws IOException { Path writtenCrate = tempDir.resolve("written-crate"); Path extractionPath = tempDir.resolve("checkMe"); + String id = "id will be encoded"; { RoCrate builtCrate = getCrateWithFileAndDir(pathToFile, pathToDir) .addDataEntity(new DataSetEntity.DataSetBuilder() @@ -43,6 +45,11 @@ default void testFilesBeingAdjusted(@TempDir Path tempDir) throws IOException { .setId("lots_of_little_files/subdir-renamed/") .build() ) + .addDataEntity(new FileEntity.FileEntityBuilder() + .setId(id) + .setLocation(pathToFile) + .build() + ) .build(); this.saveCrate(builtCrate, writtenCrate); ensureCrateIsExtractedIn(writtenCrate, extractionPath); @@ -51,6 +58,12 @@ default void testFilesBeingAdjusted(@TempDir Path tempDir) throws IOException { HelpFunctions.printFileTree(correctCrate); HelpFunctions.printFileTree(extractionPath); + // Ensure the file uses the id, not the encoded id as a file name + assertTrue( + Files.exists(extractionPath.resolve(id)), + "The file '%s' should exist, because this is the ID of the entity".formatted(id) + ); + // The actual file name should **not** appear in the crate String fileName = pathToFile.getFileName().toString(); assertFalse(