#!/usr/bin/env slsh require ("curl"); require ("pcre"); require ("cmdopt"); private variable Version = "0.3.1"; private variable Dir_Regexp = pcre_compile ("^[dl][-+rwxtsr]+ .* ([-0-9A-Z_a-z./]+)$"); private variable File_Regexp = pcre_compile ("^-[-rwxtsr+]+ .* ([-0-9A-Z_a-z.]+)$"); private define write_to_string (sp, data) { @sp = strcat (@sp, data); return 1; } private define write_to_fp (fp, data) { variable len = bstrlen (data); if (bstrlen(data) != fwrite (data, fp)) return -1; return 0; } private define get_dir_listing (c, dir) { variable data = ""; curl_setopt (c, CURLOPT_URL, path_concat (dir, "")); curl_setopt (c, CURLOPT_WRITEFUNCTION, &write_to_string, &data); variable e; try (e) { curl_perform (c); } catch CurlError: { () = fprintf (stderr, "Curl Error: %S\n", e.message); if (e.error == CURLE_LDAP_SEARCH_FAILED) () = fprintf (stderr, "FTP directory lookup failed: %S\n", dir); else () = fprintf (stderr, "The server may be down. Please try again later\n"); exit (1); } variable lines = strtok (data, "\r\n"); variable dir_list = {}; variable file_list = {}; foreach (lines) { variable line = (); if (pcre_exec (Dir_Regexp, line)) { list_append (dir_list, pcre_nth_substr (Dir_Regexp, line, 1)); continue; } if (pcre_exec (File_Regexp, line)) { list_append (file_list, pcre_nth_substr (File_Regexp, line, 1)); continue; } } return dir_list, file_list; } private define find_obsid_url (c, rooturl, obsid); private define find_obsid_url (c, rooturl, obsid) { variable dirlist, filelist; (dirlist,) = get_dir_listing (c, rooturl); % If we are searching .../cat/, then don't recurse more. variable this_dir = path_basename (path_dirname (path_concat (rooturl,""))); variable recurse_ok = strncmp (this_dir, "cat", 3); foreach (dirlist) { variable subdir = (); variable dir = path_concat (rooturl, subdir); %() = fprintf (stdout, "%s\n", dir); if (subdir == obsid) return dir; !if (recurse_ok) continue; dir = find_obsid_url (c, dir, obsid); if (dir != NULL) return dir; } return NULL; } private define get_recursive_listing (c, rooturl); private define get_recursive_listing (c, rooturl, list) { variable dirlist, filelist; (dirlist,filelist) = get_dir_listing (c, rooturl); foreach (filelist) { variable file = (); list_append (list, path_concat (rooturl, file)); } foreach (dirlist) { variable subdir = (); get_recursive_listing (c, path_concat (rooturl, subdir), list); } } private define open_output_file (file) { variable fp = fopen (file, "wb"); if (fp != NULL) return fp; () = system (sprintf ("mkdir -p %s", path_dirname (file))); return fopen (file, "wb"); } private define usage () { variable pgm = path_basename (__argv[0]); variable help = ["Version $Version Usage: $pgm [options] OBSID"$, "Options:", " -h|--help This help", " -i|--info Show files that would be downloaded", %" -c|--cat science|cal|er Catalogues to search (default is all)", " -o|--dir output-dir Output directory (default obs_OBSID)", " --download[=type-list] Types of files to download", " --show-urls Display files as URLs but do not download", " -v|--verbose Show what is happening", "", "Examples:", "Download JED's standard set of files for obs 105:", " $pgm --download 105"$, "", "Show what files would be downloaded from JED's standard set:", " $pgm --info --download 105"$, "", "Download the bias and pbk files for obs 105:", " $pgm --download=bias0,pbk0 105"$, "", "Download all files for obs 105:", " $pgm --download=all 105"$, " $pgm 105"$, "", "Download calibration obsid 61273 to cal/61273:", " $pgm --download --dir=cal/61273 61273"$, ""]; foreach help (help) () = fprintf (stderr, "%s\n", help); exit (1); } private define error_routine (msg) { () = fprintf (stderr, "%s\n", msg); usage (); } define slsh_main () { variable info_mode = 0; variable jed_set = "evt1,msk1,flt1,bpix1,stat1,asol1,mtl1,pbk0,bias0,dtf1"; variable set = NULL; variable catalogues = "science,cal,er"; variable output_dir = NULL; variable verbose = 0; variable show_urls = 0; variable opts = cmdopt_new (&error_routine); opts.add ("h|help", &usage); opts.add ("show-urls", &show_urls); opts.add ("v|verbose", &verbose); opts.add ("i|info", &info_mode); opts.add ("download", &set; type="string", optional=jed_set); opts.add ("c|cat", &catalogues; type="string"); opts.add ("o|dir", &output_dir; type="string"); variable i = opts.process (__argv, 1); if (i + 1 != __argc) usage (NULL); variable obsid = __argv[i]; if (set != NULL) { set = strlow (set); if (set == "all") set = NULL; } variable host = "cdaftp.harvard.edu"; variable c, url, cat; #iffalse foreach cat (strtok (catalogues, ",")) { url = "ftp://$host/pub/$cat/"$; vmessage ("Trying %s", url); c = curl_new (url); %curl_setopt (c, CURLOPT_VERBOSE); %curl_setopt (c, CURLOPT_FTP_USE_EPSV, 0); url = find_obsid_url (c, url, obsid); if (url != NULL) break; } #else url = sprintf ("ftp://%s/pub/byobsid/%c/%s", host, obsid[-1], obsid); c = curl_new (url); #endif if (url == NULL) { () = fprintf (stderr, "Unable to find the directory for obsid %s\n", obsid); exit (1); } variable files = {}; get_recursive_listing (c, url, files); variable regexp = NULL; if (set != NULL) { regexp = sprintf (".*_(%s)\\.fits(\\.gz)?$", strtrans (strcompress (set, ","), ",", "|")); regexp = pcre_compile (regexp); } url = path_concat (url, ""); % make sure final / is there variable len = strlen (url); if ((info_mode == 0) and (show_urls == 0)) { if (output_dir == NULL) output_dir = "obs_${obsid}"$; () = mkdir (output_dir); } foreach (files) { variable file = (); variable filename = substr (file, len+1, -1); if (regexp != NULL) { if (0 == pcre_exec (regexp, filename)) continue; } if (show_urls) { () = fprintf (stdout, "%s\n", file); continue; } if (verbose || info_mode) () = fprintf (stdout, "Downloading %s\n", file); if (info_mode) continue; filename = path_concat (output_dir, filename); variable fp = open_output_file (filename); if (fp == NULL) { () = fprintf (stderr, "Unable to open %s\n", filename); exit (1); } curl_setopt (c, CURLOPT_WRITEFUNCTION, &write_to_fp, fp); curl_setopt (c, CURLOPT_URL, file); curl_perform (c); if (-1 == fclose (fp)) { () = fprintf (stderr, "Unable to close %s -- disk full?\n", filename); exit (1); } } }