add updated CNES sync program

add GFZ ICGEM list for static models

doc/source/getting_started/Getting-Started.md

@@ -5,7 +5,11 @@ Getting Started
 - Get PO.DAAC WebDAV credentials by running `podaac_webdav.py` or logging onto [PO.DAAC Drive](https://podaac-tools.jpl.nasa.gov/drive)  
 - Run `podaac_grace_sync.py` program with your WebDAV credentials to acquire GRACE/GRACE-FO and auxiliary data  
 ```bash
-python podaac_grace_sync.py --user=<username> --directory=<path_to_grace_directory>
+python podaac_grace_sync.py --user=<username> --directory=<path_to_grace_directory> --release RL06
+```
+- Run `cnes_grace_sync.py` program to acquire CNES/GRGS GRACE/GRACE-FO data  
+```bash
+python cnes_grace_sync.py --directory=<path_to_grace_directory> --release RL05
 ```
 - If correcting for Glacial Isostatic Adjustment: have full path to data file known  
     * These can be ascii files direct from many modeling groups or a reformatted ascii/netCDF4/HDF5 file  

doc/source/index.rst

@@ -28,6 +28,7 @@ missions
     user_guide/convert_calendar_decimal.md
     user_guide/convert_harmonics.md
     user_guide/convert_julian.md
+    user_guide/cnes_grace_sync.md
     user_guide/degree_amplitude.md
     user_guide/destripe_harmonics.md
     user_guide/gauss_weights.md

doc/source/user_guide/cnes_grace_sync.md

@@ -0,0 +1,19 @@
+cnes_grace_sync.py
+==================
+
+ - Syncs GRACE/GRACE-FO and auxiliary data from the [CNES Server](http://grgs.obs-mip.fr/grace)  
+ - Syncs CNES/GRGS files for RL01/RL02/RL03/RL04/RL05
+ - Creates an index file for each data product
+
+#### Calling Sequence
+```bash
+python cnes_grace_sync.py --directory <path_to_grace_directory> --release RL05
+```
+[Source code](https://github.com/tsutterley/read-GRACE-harmonics/blob/main/scripts/cnes_grace_sync.py)
+
+#### Command Line Options
+ - `-D X`, `--directory X`: Working Data Directory
+ - `-r X`, `--release X`: GRACE/GRACE-FO Data Releases to sync (RL04,RL05)
+ - `-C`, `--clobber`: Overwrite existing data in transfer
+ - `-M X`, `--mode X`: Permission mode of directories and files synced
+ - `-l`, `--log`: Output log file

doc/source/user_guide/utilities.rst

@@ -36,6 +36,15 @@ General Methods
         `local`: path to file
 
 
+.. method:: gravity_toolkit.utilities.url_split(s)
+
+    Recursively split a url path into a list
+
+    Arguments:
+
+        `s`: url string
+
+
 .. method:: gravity_toolkit.utilities.get_unix_time(time_string, format='%Y-%m-%d %H:%M:%S')
 
     Get the Unix timestamp value for a formatted date string
@@ -74,7 +83,7 @@ General Methods
         `mtimes`: list of last modification times for items in the directory
 
 
-.. method:: gravity_toolkit.utilities.from_ftp(HOST,timeout=None,local=None,hash='',chunk=16384,verbose=False,mode=0o775)
+.. method:: gravity_toolkit.utilities.from_ftp(HOST,timeout=None,local=None,hash='',chunk=16384,verbose=False,fid=sys.stdout,mode=0o775)
 
     Download a file from a ftp host
 
@@ -94,6 +103,8 @@ General Methods
 
         `verbose`: print file transfer information
 
+        `fid`: open file object to print if verbose
+
         `mode`: permissions mode of output local file
 
 
@@ -106,7 +117,7 @@ General Methods
         `HOST`: remote http host
 
 
-.. method:: gravity_toolkit.utilities.from_http(HOST,timeout=None,local=None,hash='',chunk=16384,verbose=False,mode=0o775)
+.. method:: gravity_toolkit.utilities.from_http(HOST,timeout=None,local=None,hash='',chunk=16384,verbose=False,fid=sys.stdout,mode=0o775)
 
     Download a file from a http host
 
@@ -126,6 +137,8 @@ General Methods
 
         `verbose`: print file transfer information
 
+        `fid`: open file object to print if verbose
+
         `mode`: permissions mode of output local file
 
 
@@ -192,7 +205,7 @@ General Methods
 
 
 
-.. method:: gravity_toolkit.utilities.from_podaac(HOST,username=None,password=None,build=True,timeout=None,local=None,hash='',chunk=16384,verbose=False,mode=0o775)
+.. method:: gravity_toolkit.utilities.from_podaac(HOST,username=None,password=None,build=True,timeout=None,local=None,hash='',chunk=16384,verbose=False,fid=sys.stdout,mode=0o775)
 
     Download a file from a PO.DAAC Drive https server
 
@@ -218,4 +231,23 @@ General Methods
 
         `verbose`: print file transfer information
 
+        `fid`: open file object to print if verbose
+
         `mode`: permissions mode of output local file
+
+
+.. method:: gravity_toolkit.utilities.icgem_list(host='http://icgem.gfz-potsdam.de/tom_longtime',timeout=None,parser=lxml.etree.HTMLParser())
+
+    Parse the table of static gravity field models on the GFZ ICGEM server
+
+    Keyword arguments:
+
+        `host`: url for the GFZ ICGEM gravity field table
+
+        `timeout`: timeout in seconds for blocking operations
+
+        `parser`: HTML parser for lxml
+
+    Returns:
+
+        `colfiles`: dictionary of static file urls mapped by field name

gravity_toolkit/utilities.py

@@ -1,9 +1,11 @@
 """
 utilities.py
-Written by Tyler Sutterley (09/2020)
+Written by Tyler Sutterley (12/2020)
 Download and management utilities for syncing time and auxiliary files
 
 UPDATE HISTORY:
+    Updated 12/2020: added ICGEM list for static models
+        added file object keyword for downloads if verbose printing to file
     Updated 09/2020: copy from http and https to bytesIO object in chunks
         use netrc credentials if not entered from PO.DAAC functions
         generalize build opener function for different Earthdata instances
@@ -71,6 +73,20 @@ def get_hash(local):
     else:
         return ''
 
+#-- PURPOSE: recursively split a url path
+def url_split(s):
+    """
+    Recursively split a url path into a list
+
+    Arguments
+    ---------
+    s: url string
+    """
+    head, tail = posixpath.split(s)
+    if head in ('', posixpath.sep):
+        return tail,
+    return url_split(head) + (tail,)
+
 #-- PURPOSE: returns the Unix timestamp value for a formatted date string
 def get_unix_time(time_string, format='%Y-%m-%d %H:%M:%S'):
     """
@@ -157,7 +173,7 @@ def ftp_list(HOST,timeout=None,basename=False,pattern=None,sort=False):
 
 #-- PURPOSE: download a file from a ftp host
 def from_ftp(HOST,timeout=None,local=None,hash='',chunk=16384,
-    verbose=False,mode=0o775):
+    verbose=False,fid=sys.stdout,mode=0o775):
     """
     Download a file from a ftp host
 
@@ -172,6 +188,7 @@ def from_ftp(HOST,timeout=None,local=None,hash='',chunk=16384,
     hash: MD5 hash of local file
     chunk: chunk size for transfer encoding
     verbose: print file transfer information
+    fid: open file object to print if verbose
     mode: permissions mode of output local file
 
     Returns
@@ -200,7 +217,8 @@ def from_ftp(HOST,timeout=None,local=None,hash='',chunk=16384,
         if local and (hash != remote_hash):
             #-- print file information
             if verbose:
-                print('{0} -->\n\t{1}'.format(posixpath.join(*HOST),local))
+                args = (posixpath.join(*HOST),local)
+                print('{0} -->\n\t{1}'.format(*args), file=fid)
             #-- store bytes to file using chunked transfer encoding
             remote_buffer.seek(0)
             with open(os.path.expanduser(local), 'wb') as f:
@@ -232,7 +250,7 @@ def check_connection(HOST):
 
 #-- PURPOSE: download a file from a http host
 def from_http(HOST,timeout=None,local=None,hash='',chunk=16384,
-    verbose=False,mode=0o775):
+    verbose=False,fid=sys.stdout,mode=0o775):
     """
     Download a file from a http host
 
@@ -247,6 +265,7 @@ def from_http(HOST,timeout=None,local=None,hash='',chunk=16384,
     hash: MD5 hash of local file
     chunk: chunk size for transfer encoding
     verbose: print file transfer information
+    fid: open file object to print if verbose
     mode: permissions mode of output local file
 
     Returns
@@ -273,7 +292,8 @@ def from_http(HOST,timeout=None,local=None,hash='',chunk=16384,
         if local and (hash != remote_hash):
             #-- print file information
             if verbose:
-                print('{0} -->\n\t{1}'.format(posixpath.join(*HOST),local))
+                args = (posixpath.join(*HOST),local)
+                print('{0} -->\n\t{1}'.format(*args), file=fid)
             #-- store bytes to file using chunked transfer encoding
             remote_buffer.seek(0)
             with open(os.path.expanduser(local), 'wb') as f:
@@ -284,7 +304,6 @@ def from_http(HOST,timeout=None,local=None,hash='',chunk=16384,
         remote_buffer.seek(0)
         return remote_buffer
 
-
 #-- PURPOSE: "login" to JPL PO.DAAC Drive with supplied credentials
 def build_opener(username, password, context=ssl.SSLContext(),
     password_manager=False, get_ca_certs=False, redirect=False,
@@ -420,7 +439,7 @@ def podaac_list(HOST,username=None,password=None,build=True,timeout=None,
 
 #-- PURPOSE: download a file from a PO.DAAC Drive https server
 def from_podaac(HOST,username=None,password=None,build=True,timeout=None,
-    local=None,hash='',chunk=16384,verbose=False,mode=0o775):
+    local=None,hash='',chunk=16384,verbose=False,fid=sys.stdout,mode=0o775):
     """
     Download a file from a PO.DAAC Drive https server
 
@@ -438,6 +457,7 @@ def from_podaac(HOST,username=None,password=None,build=True,timeout=None,
     hash: MD5 hash of local file
     chunk: chunk size for transfer encoding
     verbose: print file transfer information
+    fid: open file object to print if verbose
     mode: permissions mode of output local file
 
     Returns
@@ -474,7 +494,8 @@ def from_podaac(HOST,username=None,password=None,build=True,timeout=None,
         if local and (hash != remote_hash):
             #-- print file information
             if verbose:
-                print('{0} -->\n\t{1}'.format(posixpath.join(*HOST),local))
+                args = (posixpath.join(*HOST),local)
+                print('{0} -->\n\t{1}'.format(*args), file=fid)
             #-- store bytes to file using chunked transfer encoding
             remote_buffer.seek(0)
             with open(os.path.expanduser(local), 'wb') as f:
@@ -484,3 +505,34 @@ def from_podaac(HOST,username=None,password=None,build=True,timeout=None,
         #-- return the bytesIO object
         remote_buffer.seek(0)
         return remote_buffer
+
+#-- PURPOSE: list a directory on the GFZ ICGEM https server
+def icgem_list(host='http://icgem.gfz-potsdam.de/tom_longtime',timeout=None,
+    parser=lxml.etree.HTMLParser()):
+    """
+    Parse the table of static gravity field models on the GFZ ICGEM server
+
+    Keyword arguments
+    -----------------
+    host: url for the GFZ ICGEM gravity field table
+    timeout: timeout in seconds for blocking operations
+    parser: HTML parser for lxml
+
+    Returns
+    -------
+    colfiles: dictionary of static file urls mapped by field name
+    """
+    #-- try listing from https
+    try:
+        #-- Create and submit request.
+        request = urllib2.Request(host)
+        tree = lxml.etree.parse(urllib2.urlopen(request,timeout=timeout),parser)
+    except:
+        raise Exception('List error from {0}'.format(host))
+    else:
+        #-- read and parse request for files
+        colfiles = tree.xpath('//td[@class="tom-cell-modelfile"]//a/@href')
+        #-- reduce list of files to find gfc files
+        #-- return the dict of model files mapped by name
+        return {re.findall('(.*?).gfc',posixpath.basename(f)).pop():url_split(f)
+            for i,f in enumerate(colfiles) if re.search('gfc$',f)}

notebooks/GRACE-Spatial-Maps.ipynb

@@ -202,8 +202,8 @@
     "# function for setting the data release\n",
     "def set_release(sender):\n",
     "    if (procDropdown.value == 'CNES'):\n",
-    "        drel_list = ['RL01', 'RL02', 'RL03']\n",
-    "        drel_default = 'RL03'\n",
+    "        drel_list = ['RL03', 'RL04', 'RL05']\n",
+    "        drel_default = 'RL05'\n",
     "    else:\n",
     "        drel_list = ['RL04', 'RL05', 'RL06']\n",
     "        drel_default = 'RL06'\n",
@@ -214,9 +214,9 @@
     "def set_product(sender):\n",
     "    if ((procDropdown.value == 'CNES') and (drelDropdown.value == 'RL01')):\n",
     "        dset_list = ['GAC', 'GSM']\n",
-    "    elif ((procDropdown.value == 'CNES') and (drelDropdown.value == 'RL02')):\n",
+    "    elif ((procDropdown.value == 'CNES') and drelDropdown.value in ('RL02','RL03','RL05')):\n",
     "        dset_list = ['GAA', 'GAB', 'GSM']\n",
-    "    elif ((procDropdown.value == 'CNES') and (drelDropdown.value == 'RL03')):\n",
+    "    elif ((procDropdown.value == 'CNES') and (drelDropdown.value == 'RL04')):\n",
     "        dset_list = ['GSM']\n",
     "    elif (procDropdown.value == 'CSR'):\n",
     "        dset_list = ['GAC', 'GAD', 'GSM']\n",
@@ -1049,5 +1049,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 4
-}
-
+}