validate_product

Collection of functions to compare 2 CSLC HDF5 contents and metadata

cmd_line_parser()

Command line parser

Source code in src/compass/utils/validate_product.py

def cmd_line_parser():
    """
    Command line parser
    """
    parser = argparse.ArgumentParser(
        description="""Validate reference and generated (secondary) S1 CSLC products""",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument(
        '-r', '--ref-product', type=str, dest='ref_product',
        help='Reference CSLC or static layer product (i.e., golden dataset)')
    parser.add_argument(
        '-s', '--sec-product', type=str, dest='sec_product',
        help='Secondary CSLC or static layer product to compare with reference')
    parser.add_argument('-p', '--product-type', type=str, dest='product_type',
                        choices=['CSLC', 'static_layers'],
                        default='CSLC', help='Type of file to be validated')
    return parser.parse_args()

compare_cslc_metadata(file_ref, file_sec)

Compare reference and generated CSLC metadata

Parameters:

Name	Type	Description	Default
file_ref		File path to reference metadata file (golden dataset)	required
file_sec		File path to secondary metadata file to use for comparison	required

Source code in src/compass/utils/validate_product.py

def compare_cslc_metadata(file_ref, file_sec):
    '''
    Compare reference and generated CSLC metadata
    Parameters
    ----------
    file_ref: str
        File path to reference metadata file (golden dataset)
    file_sec: str
        File path to secondary metadata file to use for comparison
    '''

    # Check if metadata files exists
    if not os.path.exists(file_ref):
        print(f'ERROR reference CSLC metadata not found: {file_ref}')
        return

    if not os.path.exists(file_sec):
        print(f'ERROR CSLC metadata not found: {file_sec}')
        return

    # Get metadata keys
    with h5py.File(file_ref, 'r') as h_ref, h5py.File(file_sec, 'r') as h_sec:
        metadata_ref = set(_get_group_item_paths(h_ref[ROOT_PATH]))
        metadata_sec = set(_get_group_item_paths(h_sec[ROOT_PATH]))

    # Intersect metadata keys
    set_ref_minus_sec = metadata_ref - metadata_sec
    set_sec_minus_ref = metadata_sec - metadata_ref

    err_str = "Metadata keys do not match.\n"
    if set_ref_minus_sec:
        err_str += f'\nReference CSLC metadata extra entries: {set_ref_minus_sec}'
    if set_sec_minus_ref:
        err_str += f'\nSecondary CSLC metadata extra entries: {set_sec_minus_ref}'

    # Check if metadata key differ
    assert not set_ref_minus_sec or not set_sec_minus_ref, err_str

compare_products(file_ref, file_sec, product_type)

Compare a reference and a newly generated (i.e., secondary) CSLC or static layer product

Parameters:

Name	Type	Description	Default
file_ref		File path to reference product (golden dataset)	required
file_sec		File path to generated product to use for comparison	required
product_type		Product type of CSLC or static_layers	required

Source code in src/compass/utils/validate_product.py

def compare_products(file_ref, file_sec, product_type):
    '''
    Compare a reference and a newly generated
    (i.e., secondary) CSLC or static layer product

    Parameters
    ----------
    file_ref: str
        File path to reference product (golden dataset)
    file_sec: str
        File path to generated product to use for comparison
    product_type: str
        Product type of CSLC or static_layers
    '''

    # Check if file paths exits
    if not os.path.exists(file_ref):
        print(f'ERROR reference {product_type} product not found: {file_ref}')
        return

    if not os.path.exists(file_sec):
        print(f'ERROR secondary {product_type} product not found: {file_sec}')
        return

    # Extract some info from reference/secondary products
    dataset_names = ['VV', 'VH', 'HH', 'HV'] if product_type == 'CSLC' else \
        ['x', 'y', 'z', 'incidence', 'local_incidence', 'heading',
         'layover_shadow_mask']
    is_static_layer = product_type == 'static_layers'
    items_ref, geotransform_ref, proj_ref = \
        _grid_info_retrieve(file_ref, dataset_names, is_static_layer)
    items_sec, geotransform_sec, proj_sec = \
        _grid_info_retrieve(file_sec, dataset_names, is_static_layer)

    # Intersect grid items found
    set_ref_minus_sec = items_ref - items_sec
    set_sec_minus_ref = items_sec - items_ref

    err_str = "Grid items do not match.\n"
    if set_ref_minus_sec:
        err_str += \
            f'\nReference {product_type} extra entries: {set_ref_minus_sec}'
    if set_sec_minus_ref:
        err_str += \
            f'\nSecondary {product_type} extra entries: {set_sec_minus_ref}'

    # Check if metadata key differ
    if set_ref_minus_sec or set_sec_minus_ref:
        print(err_str)
        return

    print(f'Comparing {product_type} projection ...')
    if not proj_ref == proj_sec:
        print(f'ERROR projection in reference {proj_ref} differs '
              f'from projection in secondary {proj_sec}')
        return

    print('Comparing geo transform arrays ...')
    if not np.array_equal(geotransform_ref, geotransform_sec):
        print(f'ERROR Reference geo transform array {geotransform_ref} differs'
              f'from secondary geo transform array {geotransform_sec}')
        return

    print('Comparing raster arrays...')
    compare_rasters = _compare_complex_slc_rasters if product_type == 'CSLC' \
        else _compare_static_layer_rasters
    compare_rasters(file_ref, file_sec, items_ref)

main()

Entrypoint of the script

Source code in src/compass/utils/validate_product.py

def main():
    '''Entrypoint of the script'''
    cmd = cmd_line_parser()

    # Check CSLC products
    compare_products(cmd.ref_product, cmd.sec_product, cmd.product_type)
    print('All CSLC product checks have passed')

    # Check CSLC metadata
    compare_cslc_metadata(cmd.ref_product, cmd.sec_product)
    print('All CSLC metadata checks have passed')