Skip to content

Commit 3c7b04f

Browse files
aalhossaryaalhossary
committed
Adding placeholder for several common hashing algorithms
The expected hashing algorithms are MD5, SHA1, and SHA256
1 parent 630bb64 commit 3c7b04f

6 files changed

Lines changed: 46 additions & 19 deletions

File tree

‎biojava-core/src/main/java/org/biojava/nbio/core/util/FileDownloadUtils.java‎

Lines changed: 40 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import java.io.FileInputStream;
2626
import java.io.FileNotFoundException;
2727
import java.io.FileOutputStream;
28+
import java.io.FilenameFilter;
2829
import java.io.IOException;
2930
import java.io.InputStream;
3031
import java.io.PrintStream;
@@ -52,6 +53,10 @@ public class FileDownloadUtils {
5253
private static final String HASH_EXT = ".hash";
5354
private static final Logger logger = LoggerFactory.getLogger(FileDownloadUtils.class);
5455

56+
public enum Hash{
57+
MD5, SHA1, SHA256, UNKNOWN
58+
}
59+
5560
/**
5661
* Copy the content of file src to dst TODO since java 1.7 this is provided
5762
* in java.nio.file.Files
@@ -177,29 +182,32 @@ public static void downloadFile(URL url, File destination) throws IOException {
177182
* @param url the remote file URL to download
178183
* @param localDestination the local file to download into
179184
* @param hashURL the URL of the hash file to download. Can be <code>null</code>.
185+
* @param hash The Hashing algorithm. Ignored if <code>hashURL</code> is <code>null</code>.
180186
*/
181-
public static void createValidationFiles(URL url, File localDestination, URL hashURL){
187+
public static void createValidationFiles(URL url, File localDestination, URL hashURL, Hash hash){
182188
try {
183189
URLConnection resourceConnection = url.openConnection();
184-
createValidationFiles(resourceConnection, localDestination, hashURL);
190+
createValidationFiles(resourceConnection, localDestination, hashURL, FileDownloadUtils.Hash.UNKNOWN);
185191
} catch (IOException e) {
186192
logger.warn("could not open connection to resource file due to exception:\n\t", e.getMessage());
187193
}
188194
}
189195
/**
190196
* Creates validation files beside a file to be downloaded.<br>
191197
* Whenever possible, for a <code>file.ext</code> file, it creates
192-
* <code>file.ext.size</code> and <code>file.hash</code> for in the same
193-
* folder where <code>file.ext</code> exists.
198+
* <code>file.ext.size</code> and <code>file.hash_XXXX</code> in the same
199+
* folder where <code>file.ext</code> exists (XXXX may be DM5, SHA1, or SHA256).
194200
* If the file connection size could not be deduced from the resourceUrlConnection
195201
* {@link URLConnection}, no size file is created.
196-
* If <code>hashURL</code> is <code>null</code>, no hash file is created.
202+
* If <code>hashURL</code> is <code>null</code>, no hash file is created.<br>
203+
* <b>N.B.</b> None of the hashing algorithms is implemented (yet), because we did not need any of them yet.
197204
* @param resourceUrlConnection the remote file URLConnection to download
198205
* @param localDestination the local file to download into
199206
* @param hashURL the URL of the hash file to download. Can be <code>null</code>.
207+
* @param hash The Hashing algorithm. Ignored if <code>hashURL</code> is <code>null</code>.
200208
* @since 6.1.1
201209
*/
202-
public static void createValidationFiles(URLConnection resourceUrlConnection, File localDestination, URL hashURL){
210+
public static void createValidationFiles(URLConnection resourceUrlConnection, File localDestination, URL hashURL, Hash hash){
203211
long size = resourceUrlConnection.getContentLengthLong();
204212
if(size == -1) {
205213
logger.warn("could not find expected file size for resource {}.", resourceUrlConnection.getURL());
@@ -217,8 +225,10 @@ public static void createValidationFiles(URLConnection resourceUrlConnection, Fi
217225
if(hashURL == null)
218226
return;
219227

228+
if(hash == Hash.UNKNOWN)
229+
throw new IllegalArgumentException("Hash URL given but algorithm is unknown");
220230
try {
221-
File hashFile = new File(localDestination.getParentFile(), localDestination.getName() + HASH_EXT);
231+
File hashFile = new File(localDestination.getParentFile(), String.format("%s%s_%s", localDestination.getName(), HASH_EXT, hash));
222232
downloadFile(hashURL, hashFile);
223233
} catch (IOException e) {
224234
logger.warn("could not write validation hash file due to exception:\n\t{}", e.getMessage());
@@ -229,12 +239,12 @@ public static void createValidationFiles(URLConnection resourceUrlConnection, Fi
229239
* Validate a local file based on pre-existing metadata files for size and hash.<br>
230240
* If the passed in <code>localFile</code> parameter is a file named <code>file.ext</code>, the function searches in the same folder for:
231241
* <ul>
232-
* <li><code>file.ext.size</code>: if found, it compares the size stored in it to the length of <code>localFile</code> (in bytes).</li>
233-
* <li><code>file.ext.hash</code>: if found, it compares the size stored in it to the hash code of <code>localFile</code>.</li>
242+
* <li><code>file.ext.size</code>: If found, it compares the size stored in it to the length of <code>localFile</code> (in bytes).</li>
243+
* <li><code>file.ext.hash_XXXX (where XXXX is DM5, SHA1, or SHA256)</code>: If found, it compares the size stored in it to the hash code of <code>localFile</code>.</li>
234244
* </ul>
235245
* If any of these comparisons fail, the function returns <code>false</code>. otherwise it returns true.
236246
* <p>
237-
* This function does not implement hash code verification yet.
247+
* <b>N.B.</b> None of the 3 common verification hashing algorithms are implement yet.
238248
* @param localFile The file to validate
239249
* @return <code>false</code> if any of the size or hash code metadata files exists but its contents does not match the expected value in the file, <code>true</code> otherwise.
240250
* @since 6.1.1
@@ -258,9 +268,26 @@ public static boolean validateFile(File localFile) {
258268
}
259269
}
260270

261-
File hashFile = new File(localFile.getParentFile(), localFile.getName() + HASH_EXT);
262-
if(hashFile.exists()) {
263-
throw new UnsupportedOperationException("Not yet implemented");
271+
File[] hashFiles = localFile.getParentFile().listFiles(new FilenameFilter() {
272+
String hashPattern = String.format("%s%s_(%s|%s|%s)", localFile.getName(), HASH_EXT, Hash.MD5, Hash.SHA1, Hash.SHA256);
273+
@Override
274+
public boolean accept(File dir, String name) {
275+
return name.matches(hashPattern);
276+
}
277+
});
278+
if(hashFiles.length > 0) {
279+
File hashFile = hashFiles[0];
280+
String name = hashFile.getName();
281+
String algo = name.substring(name.lastIndexOf('_') + 1);
282+
switch (Hash.valueOf(algo)) {
283+
case MD5:
284+
case SHA1:
285+
case SHA256:
286+
throw new UnsupportedOperationException("Not yet implemented");
287+
case UNKNOWN:
288+
default: // No need. Already checked above
289+
throw new IllegalArgumentException("Hashing algorithm not known: " + algo);
290+
}
264291
}
265292

266293
return true;

‎biojava-core/src/test/java/org/biojava/nbio/core/util/FileDownloadUtilsTest.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -202,7 +202,7 @@ void testValidationFiles() throws IOException{
202202
URL sourceUrl = new URL("https://ftp.wwpdb.org/pub/pdb/data/structures/divided/mmCIF/45/145d.cif.gz");
203203
File destFile = new File(System.getProperty("java.io.tmpdir"), "145d.cif.gz");
204204
File sizeFile = new File(destFile.getParentFile(), destFile.getName()+".size");
205-
File hashFile = new File(destFile.getParentFile(), destFile.getName()+".hash");
205+
File hashFile = new File(destFile.getParentFile(), destFile.getName()+".hash_MD5");
206206
System.out.println(destFile.getAbsolutePath());
207207
destFile.delete();
208208
sizeFile.delete();
@@ -222,7 +222,7 @@ void testValidationFiles() throws IOException{
222222
assertFalse(FileDownloadUtils.validateFile(destFile), "file not detected to be invalid although size value is wrong.");
223223
System.out.println("Just ignore the previous warning. It is expected.");
224224

225-
FileDownloadUtils.createValidationFiles(sourceUrl, destFile, null);
225+
FileDownloadUtils.createValidationFiles(sourceUrl, destFile, null, FileDownloadUtils.Hash.UNKNOWN);
226226
assertTrue(sizeFile.exists(), "couldn't create size file");
227227
assertTrue(FileDownloadUtils.validateFile(destFile), "file not detected to be invalid although there is correct size validation file");
228228

‎biojava-structure/src/main/java/org/biojava/nbio/structure/ecod/EcodInstallation.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -406,7 +406,7 @@ private void downloadDomains() throws IOException {
406406
File localFile = getDomainFile();
407407

408408
logger.info("Downloading {} to: {}",domainsURL, localFile);
409-
FileDownloadUtils.createValidationFiles(domainsURL, localFile, null);
409+
FileDownloadUtils.createValidationFiles(domainsURL, localFile, null, FileDownloadUtils.Hash.UNKNOWN);
410410
FileDownloadUtils.downloadFile(domainsURL, localFile);
411411
if(! FileDownloadUtils.validateFile(localFile))
412412
throw new IOException("Downloaded file invalid: "+ localFile);

‎biojava-structure/src/main/java/org/biojava/nbio/structure/io/LocalPDBDirectory.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -581,7 +581,7 @@ private File downloadStructure(PdbId pdbId, String pathOnServer, boolean obsolet
581581
logger.info("Fetching " + ftp);
582582
logger.info("Writing to "+ realFile);
583583

584-
FileDownloadUtils.createValidationFiles(url, realFile, null);
584+
FileDownloadUtils.createValidationFiles(url, realFile, null, FileDownloadUtils.Hash.UNKNOWN);
585585
FileDownloadUtils.downloadFile(url, realFile);
586586
if(! FileDownloadUtils.validateFile(realFile))
587587
throw new IOException("Downloaded file invalid: "+realFile);

‎biojava-structure/src/main/java/org/biojava/nbio/structure/io/sifts/SiftsMappingProvider.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ public static List<SiftsEntity> getSiftsMapping(String pdbId) throws IOException
8888
String u = String.format(fileLoc,pdbId);
8989
URL url = new URL(u);
9090
logger.debug("Downloading SIFTS file {} validation metadata.",url);
91-
FileDownloadUtils.createValidationFiles(url, dest, null);
91+
FileDownloadUtils.createValidationFiles(url, dest, null, FileDownloadUtils.Hash.UNKNOWN);
9292
logger.debug("Downloading SIFTS file {} to {}",url,dest);
9393
FileDownloadUtils.downloadFile(url, dest);
9494
}

‎biojava-structure/src/main/java/org/biojava/nbio/structure/scop/ScopInstallation.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -747,7 +747,7 @@ protected void downloadComFile() throws FileNotFoundException, IOException{
747747
*/
748748
protected void downloadFileFromRemote(URL remoteURL, File localFile) throws IOException{
749749
logger.info("Downloading " + remoteURL + " to: " + localFile);
750-
FileDownloadUtils.createValidationFiles(remoteURL, localFile, null);
750+
FileDownloadUtils.createValidationFiles(remoteURL, localFile, null, FileDownloadUtils.Hash.UNKNOWN);
751751
FileDownloadUtils.downloadFile(remoteURL, localFile);
752752
if(! FileDownloadUtils.validateFile(localFile))
753753
throw new IOException("Downloaded file invalid: "+localFile);

0 commit comments

Comments
 (0)