s1_geocode_stack

create_runconfig(burst_map_row, dem_file, work_dir, flatten, pol, x_spac, y_spac, enable_corrections, burst_db_file)

Create runconfig to process geocoded bursts

Parameters:

Name	Type	Description	Default
burst_map_row		one row from the dataframe method burst_map.itertuples()	required
dem_file		Path to DEM to use for processing	required
work_dir		Path to working directory for temp and final results	required
flatten		Flag to enable/disable flattening	required
pol		Polarizations to process. Choices: co-pol, cross-pol, dual-pol	required
x_spac		Spacing of geocoded burst along X-direction	required
y_spac		Spacing of geocoded burst along Y-direction	required
enable_corrections		Flag to enable/disable applying corrections to burst stacks.	required
burst_db_file		Path to burst database file to use for burst bounding boxes.	required

Returns:

Name	Type	Description
runconfig	str	Path to runconfig file

Source code in src/compass/s1_geocode_stack.py

def create_runconfig(burst_map_row, dem_file, work_dir, flatten, pol, x_spac,
                     y_spac, enable_corrections, burst_db_file):
    """
    Create runconfig to process geocoded bursts

    Parameters
    ----------
    burst_map_row: namedtuple
        one row from the dataframe method `burst_map.itertuples()`
    dem_file: str
        Path to DEM to use for processing
    work_dir: str
        Path to working directory for temp and final results
    flatten: bool
        Flag to enable/disable flattening
    pol: str
        Polarizations to process. Choices: co-pol, cross-pol, dual-pol
    x_spac: float
        Spacing of geocoded burst along X-direction
    y_spac: float
        Spacing of geocoded burst along Y-direction
    enable_corrections: bool
        Flag to enable/disable applying corrections to burst stacks.
    burst_db_file: str
        Path to burst database file to use for burst bounding boxes.

    Returns
    -------
    runconfig: str
        Path to runconfig file
    """
    # Load default runconfig and fill it with user-defined options
    yaml_path = f'{helpers.WORKFLOW_SCRIPTS_DIR}/defaults/s1_cslc_geo.yaml'
    with open(yaml_path, 'r') as stream:
        yaml_cfg = yaml.safe_load(stream)

    groups = yaml_cfg['runconfig']['groups']
    inputs = groups['input_file_group']
    product = groups['product_path_group']
    process = groups['processing']
    geocode = process['geocoding']

    # Allocate Inputs
    burst = burst_map_row.burst
    inputs['safe_file_path'] = [burst_map_row.zip_file]
    inputs['orbit_file_path'] = [burst_map_row.orbit_path]
    inputs['burst_id'] = [str(burst.burst_id)]
    groups['dynamic_ancillary_file_group']['dem_file'] = dem_file
    groups['static_ancillary_file_group']['burst_database_file'] = burst_db_file

    # Product path
    product['product_path'] = work_dir
    product['scratch_path'] = f'{work_dir}/scratch'
    product['sas_output_file'] = work_dir

    # Geocoding
    process['polarization'] = pol
    process['correction_luts']['enabled'] = enable_corrections
    geocode['flatten'] = flatten
    geocode['x_posting'] = x_spac
    geocode['y_posting'] = y_spac

    date_str = burst.sensing_start.strftime("%Y%m%d")
    os.makedirs(f'{work_dir}/runconfigs', exist_ok=True)
    runconfig_path = f'{work_dir}/runconfigs/geo_runconfig_{date_str}_{str(burst.burst_id)}.yaml'
    with open(runconfig_path, 'w') as yaml_file:
        yaml.dump(yaml_cfg, yaml_file, default_flow_style=False)
    return runconfig_path

generate_burst_map(zip_files, orbit_dir, output_epsg=None, bbox=None, bbox_epsg=4326, burst_db_file=DEFAULT_BURST_DB_FILE)

Generates a dataframe of geogrid infos for each burst ID in zip_files.

Parameters:

Name	Type	Description	Default
zip_files		List of S1-A/B SAFE (zip) files	required
orbit_dir		Directory containing sensor orbit ephemerides	required
output_epsg		EPSG code identifying output product projection system	None
bbox		Desired bounding box of the geocoded bursts as (left, bottom, right, top). If not provided, the bounding box is computed for each burst.	None
bbox_epsg		EPSG code of the bounding box. If 4326, the bounding box is assumed to be lon/lat degrees (default: 4326).	4326
burst_db_file		Path to the burst database file to load bounding boxes.	DEFAULT_BURST_DB_FILE

Returns:

Name	Type	Description
burst_map	Dataframe	Pandas dataframe containing geogrid info (e.g. top-left, bottom-right x and y coordinates) for each burst to process

Source code in src/compass/s1_geocode_stack.py

def generate_burst_map(zip_files, orbit_dir, output_epsg=None, bbox=None,
                       bbox_epsg=4326, burst_db_file=DEFAULT_BURST_DB_FILE):
    """Generates a dataframe of geogrid infos for each burst ID in `zip_files`.

    Parameters
    ----------
    zip_files: str
        List of S1-A/B SAFE (zip) files
    orbit_dir: str
        Directory containing sensor orbit ephemerides
    output_epsg: int
        EPSG code identifying output product projection system
    bbox: Optional[tuple[float]]
        Desired bounding box of the geocoded bursts as (left, bottom, right, top).
        If not provided, the bounding box is computed for each burst.
    bbox_epsg: int
        EPSG code of the bounding box. If 4326, the bounding box is assumed
        to be lon/lat degrees (default: 4326).
    burst_db_file: str
        Path to the burst database file to load bounding boxes.

    Returns
    -------
    burst_map: pandas.Dataframe
        Pandas dataframe containing geogrid info (e.g. top-left, bottom-right
        x and y coordinates) for each burst to process
    """
    # Initialize dictionary that contains all the info for geocoding
    burst_map = defaultdict(list)

    # Get all the bursts from safe file
    i_subswath = [1, 2, 3]

    for zip_file in zip_files:
        orbit_path = get_orbit_file_from_dir(zip_file, orbit_dir, auto_download=True)

        for subswath in i_subswath:
            ref_bursts = load_bursts(zip_file, orbit_path, subswath)
            for burst in ref_bursts:
                epsg, bbox_utm = _get_burst_epsg_and_bbox(
                    burst, output_epsg, bbox, bbox_epsg, burst_db_file
                )
                if epsg is None:  # Flag for skipping burst
                    continue

                burst_map['burst_id'].append(str(burst.burst_id))
                # keep the burst object so we don't have to re-parse
                burst_map['burst'].append(burst)

                burst_map['date'].append(burst.sensing_start.strftime("%Y%m%d"))
                # Save the file paths for creating the runconfig
                burst_map['orbit_path'].append(orbit_path)
                burst_map['zip_file'].append(zip_file)

    burst_map = pd.DataFrame(data=burst_map)
    return burst_map

get_common_burst_ids(data)

Get list of burst IDs common among all processed dates

Parameters:

Name	Type	Description	Default
data		Dataframe containing info for stitching (e.g. burst IDs)	required

Returns:

Name	Type	Description
common_id	list	List containing common burst IDs among all the dates

Source code in src/compass/s1_geocode_stack.py

def get_common_burst_ids(data):
    """Get list of burst IDs common among all processed dates

    Parameters
    ----------
    data: pandas.DataFrame
        Dataframe containing info for stitching (e.g. burst IDs)

    Returns
    -------
    common_id: list
        List containing common burst IDs among all the dates
    """
    # Identify all the dates for the bursts to stitch
    unique_dates = list(set(data['date']))

    # Initialize list of unique burst IDs
    common_id = data.burst_id[data.date == unique_dates[0]]

    for date in unique_dates:
        ids = data.burst_id[data.date == date]
        common_id = sorted(list(set(ids.tolist()) & set(common_id)))
    return common_id

main()

Create the command line interface and run the script.

Source code in src/compass/s1_geocode_stack.py

def main():
    """Create the command line interface and run the script."""
    # Run main script
    args = create_parser()

    run(
        slc_dir=args.slc_dir,
        dem_file=args.dem_file,
        burst_id=args.burst_id,
        common_bursts_only=args.common_bursts_only,
        start_date=args.start_date,
        end_date=args.end_date,
        exclude_dates=args.exclude_dates,
        orbit_dir=args.orbit_dir,
        work_dir=args.work_dir,
        pol=args.pol,
        x_spac=args.x_spac,
        y_spac=args.y_spac,
        bbox=args.bbox,
        bbox_epsg=args.bbox_epsg,
        output_epsg=args.output_epsg,
        burst_db_file=args.burst_db_file,
        flatten=not args.no_flatten,
        enable_corrections=not args.no_corrections,
        using_zipped=not args.unzipped,
    )

prune_dataframe(data, id_col, id_list)

Prune dataframe based on column ID and list of value

Parameters:

data: pandas.DataFrame dataframe that needs to be pruned id_col: str column identification for 'data' (e.g. 'burst_id') id_list: list List of elements to consider when pruning. If exclude_items is False (default), then all elements in data will be kept except for those in id_list. If exclude_items is True, the items in id_list will be removed from data.

Returns:

data: pandas.DataFrame Pruned dataframe with rows in 'id_list'

Source code in src/compass/s1_geocode_stack.py

def prune_dataframe(data, id_col, id_list):
    """Prune dataframe based on column ID and list of value

    Parameters:
    ----------
    data: pandas.DataFrame
        dataframe that needs to be pruned
    id_col: str
        column identification for 'data' (e.g. 'burst_id')
    id_list: list
        List of elements to consider when pruning.
        If exclude_items is False (default), then all elements in `data`
            will be kept *except for* those in `id_list`.
        If exclude_items is True, the items in `id_list` will be removed from `data`.

    Returns:
    -------
    data: pandas.DataFrame
        Pruned dataframe with rows in 'id_list'
    """
    pattern = '|'.join(id_list)
    df = data.loc[data[id_col].str.contains(pattern, case=False)]
    return df

run(slc_dir, dem_file, burst_id=None, common_bursts_only=False, start_date=None, end_date=None, exclude_dates=None, orbit_dir=None, work_dir='stack', pol='co-pol', x_spac=5, y_spac=10, bbox=None, bbox_epsg=4326, output_epsg=None, burst_db_file=DEFAULT_BURST_DB_FILE, flatten=True, enable_corrections=True, using_zipped=True)

Create runconfigs and runfiles generating geocoded bursts for a static stack of Sentinel-1 A/B SAFE files.

Parameters:

Name	Type	Description	Default
slc_dir		Directory containing S1-A/B SAFE files	required
dem_file		File path to DEM to use for processing	required
burst_id		List of burst IDs to process (default: None)	None
common_bursts_only		Flag to only process bursts common to all SAFE files (default: False)	False
start_date		Date of the start acquisition of the stack (format: YYYYMMDD)	None
end_date		Date of the end acquisition of the stack (format: YYYYMMDD)	None
exclude_dates		List of dates to exclude from the stack (format: YYYYMMDD)	None
orbit_dir		Directory containing orbit files	None
work_dir		Working directory to store temp and final files	'stack'
pol		Polarization to process (default: co-pol).	'co-pol'
x_spac		Spacing of geocoded burst along X-direction. Default: 5 (meters)	5
y_spac		Spacing of geocoded burst along Y-direction. Default: 10 (meters)	10
bbox		Bounding box of the area to geocode: (xmin, ymin, xmax, ymax) in degrees. Used to filter bursts which do not overlap.	None
bbox_epsg		EPSG code of the bounding box coordinates (default: 4326) If using EPSG:4326, the bounding box coordinates are in degrees.	4326
output_epsg		EPSG code identifying projection system to use for output. If not specified, will search for each burst center's EPSG from the burst database.	None
burst_db_file	str	File path to burst database containing EPSG/extent information.	DEFAULT_BURST_DB_FILE
flatten		Enable/disable flattening (removal of the DEM phase) of geocoded burst.	True
enable_corrections		Enable/disable generation/usage of correction LUTs during geocoding.	True
using_zipped		Flag to indicate if SAFE files are zipped or not (default: True). Will search for .zip files if True, and .SAFE directories if False.	True

Source code in src/compass/s1_geocode_stack.py

def run(slc_dir, dem_file, burst_id=None, common_bursts_only=False, start_date=None,
        end_date=None, exclude_dates=None, orbit_dir=None, work_dir='stack',
        pol='co-pol', x_spac=5, y_spac=10, bbox=None, bbox_epsg=4326,
        output_epsg=None, burst_db_file=DEFAULT_BURST_DB_FILE, flatten=True,
        enable_corrections=True, using_zipped=True):
    """Create runconfigs and runfiles generating geocoded bursts for a static
    stack of Sentinel-1 A/B SAFE files.

    Parameters
    ----------
    slc_dir: str
        Directory containing S1-A/B SAFE files
    dem_file: str
        File path to DEM to use for processing
    burst_id: Optional[list]
        List of burst IDs to process (default: None)
    common_bursts_only: bool
        Flag to only process bursts common to all SAFE files (default: False)
    start_date: str
        Date of the start acquisition of the stack (format: YYYYMMDD)
    end_date: str
        Date of the end acquisition of the stack (format: YYYYMMDD)
    exclude_dates: list[str]
        List of dates to exclude from the stack (format: YYYYMMDD)
    orbit_dir: str
        Directory containing orbit files
    work_dir: str
        Working directory to store temp and final files
    pol: str, choices=['co-pol', 'dual-pol', 'cross-pol']
        Polarization to process (default: co-pol).
    x_spac: float
        Spacing of geocoded burst along X-direction. Default: 5 (meters)
    y_spac: float
        Spacing of geocoded burst along Y-direction. Default: 10 (meters)
    bbox: tuple[float], optional
        Bounding box of the area to geocode: (xmin, ymin, xmax, ymax) in degrees.
        Used to filter bursts which do not overlap.
    bbox_epsg: int
        EPSG code of the bounding box coordinates (default: 4326)
        If using EPSG:4326, the bounding box coordinates are in degrees.
    output_epsg: int
        EPSG code identifying projection system to use for output.
        If not specified, will search for each burst center's EPSG from
        the burst database.
    burst_db_file : str
        File path to burst database containing EPSG/extent information.
    flatten: bool
        Enable/disable flattening (removal of the DEM phase) of geocoded burst.
    enable_corrections: bool
        Enable/disable generation/usage of correction LUTs during geocoding.
    using_zipped: bool
        Flag to indicate if SAFE files are zipped or not (default: True).
        Will search for .zip files if True, and .SAFE directories if False.
    """
    start_time = time.perf_counter()
    error = journal.error('s1_geo_stack_processor.main')
    info = journal.info('s1_geo_stack_processor.main')

    # Check if SLC dir and DEM exists
    if not os.path.isdir(slc_dir):
        err_str = f'{slc_dir} SLC directory does not exist'
        error.log(err_str)
        raise FileNotFoundError(err_str)

    if not os.path.isfile(dem_file):
        err_str = f'{dem_file} DEM file does not exists'
        error.log(err_str)
        raise FileNotFoundError(err_str)

    # Create directory for runfiles
    run_dir = f'{work_dir}/run_files'
    os.makedirs(run_dir, exist_ok=True)

    # Check if orbit are provided, if Not download
    if orbit_dir is None:
        orbit_dir = f'{work_dir}/orbits'
        info.log(f'Orbit directory not assigned. Using {orbit_dir} to download orbits')
        os.makedirs(orbit_dir, exist_ok=True)
        # Note: Specific files will be downloaded as needed during `generate_burst_map`

    # Generate burst map and prune it if a list of burst ID is provided
    search_ext = 'zip' if using_zipped else 'SAFE'
    zip_file_list = sorted(glob.glob(f'{slc_dir}/S1[AB]_*.{search_ext}'))
    # Remove zip files that are not in the date range before generating burst map
    zip_file_list = _filter_by_date(zip_file_list, start_date, end_date, exclude_dates)

    info.log(f'Generating burst map for {len(zip_file_list)} SAFE files')
    burst_map = generate_burst_map(
        zip_file_list, orbit_dir, output_epsg, bbox, bbox_epsg, burst_db_file
    )

    # Identify burst IDs common across the stack and remove from the dataframe
    # burst IDs that are not in common
    if common_bursts_only:
        common_ids = get_common_burst_ids(burst_map)
        burst_map = prune_dataframe(burst_map, 'burst_id', common_ids)

    # If user selects burst IDs to process, prune unnecessary bursts
    if burst_id is not None:
        burst_map = prune_dataframe(burst_map, 'burst_id', burst_id)

    # Ready to geocode bursts
    for row in burst_map.itertuples():
        runconfig_path = create_runconfig(
            row,
            dem_file=dem_file,
            work_dir=work_dir,
            flatten=flatten,
            pol=pol,
            x_spac=x_spac,
            y_spac=y_spac,
            enable_corrections=enable_corrections,
            burst_db_file=burst_db_file,
        )
        date_str = row.burst.sensing_start.strftime("%Y%m%d")
        runfile_name = f'{run_dir}/run_{date_str}_{row.burst.burst_id}.sh'
        with open(runfile_name, 'w') as rsh:
            path = os.path.dirname(os.path.realpath(__file__))
            rsh.write(
                f'python {path}/s1_cslc.py {runconfig_path}\n')

    end_time = time.perf_counter()
    print('Elapsed time (min):', (end_time - start_time) / 60.0)