Little script for combining WGET files

I’ve been using CMIP5 data in my research, but it’s a pain to download, especially if you want specific variables from specific datasets. Simply download the WGET scripts containing files you need, then build this script in Java 7+. It’ll combine make a big list of all the files of interest. Put those files in another WGET script and download away!


import java.io.BufferedReader;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.charset.Charset;
import java.io.File;
import java.util.ArrayList;

public class ConcatenateWGET {

public static void main(String[] args) {
File folder = new File("C:/Users/pende/Desktop/Harvard/Daniel Jacob/Siberian High analysis/WGET scripts/actually wget");
File[] listOfFiles = folder.listFiles();

BufferedReader br = null;
String line = "";
ArrayList commands = new ArrayList();

for(File f : listOfFiles) {
int counter = 0;
try {
br = new BufferedReader(new FileReader(f));
while ((line = br.readLine()) != null) {
counter++;
if(counter>27){
if(line.charAt(0)!='\'') {break;}
String type = line.split("_")[0].substring(1);
if(type.equals("rhs")||type.equals("va")||type.equals("ua")||type.equals("ta")) {
commands.add(line);
}
}
}
} catch(Exception e) {
System.out.println();
} finally {
if (br != null) {
try {
br.close();
} catch (IOException e) {

}
}
}
}

try {
Path file = Paths.get("wget-list.txt");
Files.write(file, commands, Charset.forName("UTF-8"));
} catch(Exception e) {
System.out.println();
}
}
}

 

Leave a Reply

Your email address will not be published. Required fields are marked *