// get all tables that have DLP findings
public List<JsonMessage> processDatasets(List<String> datasetIncludeList,
                                             List<String> datasetExcludeList,
                                             List<String> tableExcludeList,
                                             String dataRegionId) throws IOException, InterruptedException, NonRetryableApplicationException {
        List<String> tablesIncludeList = new ArrayList<>();
        for (String dataset : datasetIncludeList) {
            try {
                if (!datasetExcludeList.contains(dataset)) {
                    List<String> tokens = Utils.tokenize(dataset, ".", true);
                    String projectId = tokens.get(0);
                    String datasetId = tokens.get(1);
                    String datasetLocation = bqService.getDatasetLocation(projectId, datasetId);
                /*
                 TODO: Support tagging in multiple locations
                 to support all locations:
                 1- Taxonomies/PolicyTags have to be created in each required location
                 2- Update the Tagger Cloud Function to read one mapping per location
                 For now, we don't submit tasks for tables in other locations than the PolicyTag location
                 */
                    if (!datasetLocation.toLowerCase().equals(dataRegionId.toLowerCase())) {
                        logger.logWarnWithTracker(runId,
                                String.format(
                                        "Ignoring dataset %s in location %s. Only location %s is configured",
                                        dataset,
                                        datasetLocation,
                                        dataRegionId)
                        );
                        continue;
                    }
                    // get all tables that have DLP findings
                    List<String> datasetTables = scanner.listChildren(projectId, datasetId);
                    tablesIncludeList.addAll(datasetTables);
                    if (datasetTables.isEmpty()) {
                        String msg = String.format(
                                "No Tables found under dataset '%s'",
                                dataset);
                        logger.logWarnWithTracker(runId, msg);
                    } else {
                        logger.logInfoWithTracker(runId, String.format("Tables found in dataset %s : %s", dataset, datasetTables));
                    }
                }
            } catch (Exception exception) {
                // log and continue
                logger.logFailedDispatcherEntityId(runId, dataset, exception);
            }
        }
        return processTables(tablesIncludeList, tableExcludeList);
    }
continue;
}
// get all tables that have DLP findings
List<String> datasetTables = scanner.listChildren(projectId, datasetId);
tablesIncludeList.addAll(datasetTables);
if (!datasetLocation.toLowerCase().equals(dataRegionId.toLowerCase())) {
logger.logWarnWithTracker(runId,
String.format(
"Ignoring dataset %s in location %s. Only location %s is configured",
dataset,
datasetLocation,
dataRegionId)
);
continue;
}
// get all tables that have DLP findings
List<String> datasetTables = scanner.listChildren(projectId, datasetId);
tablesIncludeList.addAll(datasetTables);
if (datasetTables.isEmpty()) {
String msg = String.format(
"No Tables found under dataset '%s'",
dataset);
logger.logWarnWithTracker(runId, msg);
} else {
logger.logInfoWithTracker(runId, String.format("Tables found in dataset %s : %s", dataset, datasetTables));
}
String projectId = tokens.get(0);
String datasetId = tokens.get(1);
String datasetLocation = bqService.getDatasetLocation(projectId, datasetId);
/*
TODO: Support tagging in multiple locations
to support all locations:
1- Taxonomies/PolicyTags have to be created in each required location
2- Update the Tagger Cloud Function to read one mapping per location
For now, we don't submit tasks for tables in other locations than the PolicyTag location
*/
if (!datasetLocation.toLowerCase().equals(dataRegionId.toLowerCase())) {
logger.logWarnWithTracker(runId,
String.format(
"Ignoring dataset %s in location %s. Only location %s is configured",
dataset,
datasetLocation,
dataRegionId)
);
continue;
}
// get all tables that have DLP findings
List<String> datasetTables = scanner.listChildren(projectId, datasetId);
tablesIncludeList.addAll(datasetTables);
if (datasetTables.isEmpty()) {
String msg = String.format(
"No Tables found under dataset '%s'",
dataset);
logger.logWarnWithTracker(runId, msg);
} else {
logger.logInfoWithTracker(runId, String.format("Tables found in dataset %s : %s", dataset, datasetTables));
}
}
} catch (Exception exception) {
// log and continue
logger.logFailedDispatcherEntityId(runId, dataset, exception);
}
}
return processTables(tablesIncludeList, tableExcludeList);
}